Introduction

Submitted by: Susan Bataju

For this Lab, "Tomato leaf disease detection" image dataset by Kaustubh B. was selected from Kaggle[1]. The dataset has ten different types of diseases for tomato leaves, they are listed below. For each type of disease there are 10000 images in train folder and 1000 images in test folder. The task is to carryout multiclass classification. Here is the list of the diesease:

  • Tomatomosaicvirus
  • Target_Spot
  • Bacterial_spot
  • TomatoYellowLeafCurlVirus
  • Late_blight
  • Leaf_Mold
  • Early_blight
  • Spidermites Two-spottedspider_mite
  • Tomato___healthy
  • Septorialeafspot

Preparation

All the images in the dataset are of same shape and imported as 256x256x3 numpy arrays. Then, for the first part all images are reshaped as 128x128x3 arrays later into 224x224x3. As the dataset is already split in 10000 train and 1000 test, I further divide train set into 9000 actual train set and 1000 validataion set. It would mean that validation set and test set are equal in size. Lastly all the class have equal number of images.



In [1]:
import tensorflow as tf
print("Num of GPUs available: ", len(tf.test.gpu_device_name()))
# Num of GPUs available:  13
import os
Num of GPUs available:  13
2022-12-04 12:28:50.849202: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  SSE4.1 SSE4.2 AVX AVX2 FMA
To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
2022-12-04 12:28:51.836299: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1510] Created device /device:GPU:0 with 15389 MB memory:  -> device: 0, name: Tesla P100-PCIE-16GB, pci bus id: 0000:06:00.0, compute capability: 6.0
In [2]:
import cv2
%matplotlib inline
from matplotlib import pyplot as plt
import numpy as np
import os
from sklearn import metrics as mt
import matplotlib.image as mpimg
from keras.models import Sequential
from keras.layers.convolutional import Conv2D
from keras.layers.convolutional import MaxPooling2D
from keras.layers.core import Activation, Flatten, Dropout, Dense
from tensorflow.keras.layers import BatchNormalization
from sklearn.model_selection import train_test_split
from keras.preprocessing.image import ImageDataGenerator
from sklearn.preprocessing import LabelBinarizer
# import seaborn as sns
from skimage.io import imshow
from sklearn.metrics import plot_confusion_matrix
from tensorflow.keras.utils import plot_model
from skimage.transform import resize
# import pandas as pd
In [3]:
import tensorflow.keras as keras
from tensorflow.keras import layers
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Reshape, Input
from tensorflow.keras.layers import Dense, Dropout, Activation, Flatten
from tensorflow.keras.layers import Conv2D, MaxPooling2D
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.callbacks import EarlyStopping
from tensorflow.keras.regularizers import l2
from tensorflow.keras.layers import average 
from tensorflow.keras.models import  Model
In [4]:
from PIL import Image
plt.style.use('ggplot')
In [5]:
def plot_history(history):
    acc = history.history['accuracy']
    val_acc = history.history['val_accuracy']

    loss = history.history['loss']
    val_loss = history.history['val_loss']

    epochs_range = range(len(acc))

    plt.figure(figsize=(8, 8))
    plt.subplot(1, 2, 1)
    plt.plot(epochs_range, acc, label='Training Accuracy')
    plt.plot(epochs_range, val_acc, label='Validation Accuracy')
    plt.legend(loc='lower right')
    plt.title('Training and Validation Accuracy')

    plt.subplot(1, 2, 2)
    plt.plot(epochs_range, loss, label='Training Loss')
    plt.plot(epochs_range, val_loss, label='Validation Loss')
    plt.legend(loc='upper right')
    plt.title('Training and Validation Loss')
    plt.show()

#https://stackoverflow.com/a/29877565/20626850 

def plot_confusion_matrix(df_confusion,label, title='Confusion matrix', cmap=plt.cm.winter):
    df_confusion = df_confusion/sum(df_confusion[0])
    plt.matshow(df_confusion, cmap=cmap,fignum=10) # imshow
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(label))
    plt.xticks(tick_marks, label, rotation=90)
    plt.yticks(tick_marks,label)
    for (i, j), z in np.ndenumerate(df_confusion):
        plt.text(j, i, '{:0.2f}'.format(z), ha='center', va='center',color='black')

#     plt.tight_layout()
    plt.grid()
    plt.ylabel('Actual')
    plt.xlabel('Predicted')

    
In [6]:
def img_to_np(DIR,flatten=True,newsize = (128,128)):

    img = Image.open(DIR)
    img = img.resize(newsize)
    return np.asarray(img)

Importing the  dataset, resizing the images and reading the labels.


In [7]:
TEST_DIR="/users/sbataju/CS5324/tomato/val/"

label={}
index=0
for FOLDER in os.listdir(TEST_DIR):
#     print(FOLDER,index)
    label[index]=FOLDER
    index=index+1
inv_label = {v: k for k, v in label.items()}

NUM_CLASSES = 10
index=0
data=[]
for FOLDER in os.listdir(TEST_DIR):
    print(FOLDER,inv_label[FOLDER])
    for image_dir in os.listdir(TEST_DIR+FOLDER):
        data.append({"x":img_to_np(TEST_DIR+FOLDER+"/"+image_dir),"y":label[inv_label[FOLDER]]})
#         assert False
    index=index+1
    
x,y=[],[]
for obj in data:
    x.append(obj["x"])
    y.append(obj["y"])
x_test = np.array(x)
y_test = np.array([inv_label[i] for i in y],dtype=np.float16)  
Tomato___Tomato_mosaic_virus 0
Tomato___Spider_mites Two-spotted_spider_mite 1
Tomato___Leaf_Mold 2
Tomato___Early_blight 3
Tomato___Target_Spot 4
Tomato___Late_blight 5
Tomato___healthy 6
Tomato___Bacterial_spot 7
Tomato___Tomato_Yellow_Leaf_Curl_Virus 8
Tomato___Septoria_leaf_spot 9
In [8]:
TRAIN_DIR = "/users/sbataju/CS5324/tomato/train/"
index=0
data=[]
for FOLDER in os.listdir(TRAIN_DIR):
    print(FOLDER,'  ',inv_label[FOLDER])
    for image_dir in os.listdir(TRAIN_DIR+FOLDER):
      data.append({"x":img_to_np(TRAIN_DIR+FOLDER+"/"+image_dir,flatten=False),"y":label[inv_label[FOLDER]]})
    index=index+1
x,y=[],[]
for obj in data:
    x.append(obj["x"])
    y.append(obj["y"])
x_train = np.array(x)
y_train = np.array([inv_label[i] for i in y],dtype=np.float16)  

y_train_ohe = keras.utils.to_categorical(y_train, NUM_CLASSES)
y_test_ohe = keras.utils.to_categorical(y_test, NUM_CLASSES)
Tomato___Tomato_mosaic_virus    0
Tomato___healthy    6
Tomato___Tomato_Yellow_Leaf_Curl_Virus    8
Tomato___Septoria_leaf_spot    9
Tomato___Late_blight    5
Tomato___Early_blight    3
Tomato___Leaf_Mold    2
Tomato___Spider_mites Two-spotted_spider_mite    1
Tomato___Target_Spot    4
Tomato___Bacterial_spot    7

Plotting random 25 images.

In [9]:
plt.figure(figsize=(15,15))
for i in range(1,26):
    idx = np.random.randint(0,len(y))
    plt.subplot(5,5,i)
    plt.imshow(x[idx])
    plt.title(" ".join(y[idx].split('_')[1:]))
    plt.grid()
plt.show()
#     print(i)

Split the the train set into train and validatation dataset. As stated before 10% of the toal train data is used as validation which has same size as test set. Throughout the model testing below, this split is used partly motivated because it would be a baseline to many variations and changes which also makes it difficult to keep track of Cross Validatation if CV was used in all of them.  

In [10]:
from sklearn.model_selection import train_test_split
X_train, X_val, Y_train, Y_val = train_test_split(
    x_train, y_train_ohe, test_size=0.1, random_state=42,stratify=y_train,)
In [11]:
label
Out[11]:
{0: 'Tomato___Tomato_mosaic_virus',
 1: 'Tomato___Spider_mites Two-spotted_spider_mite',
 2: 'Tomato___Leaf_Mold',
 3: 'Tomato___Early_blight',
 4: 'Tomato___Target_Spot',
 5: 'Tomato___Late_blight',
 6: 'Tomato___healthy',
 7: 'Tomato___Bacterial_spot',
 8: 'Tomato___Tomato_Yellow_Leaf_Curl_Virus',
 9: 'Tomato___Septoria_leaf_spot'}

Here it can be seen the number of images in each class in each dataset. It can be seen, all the class have equal number of images and the number of images in each dataset as well. 

In [12]:
plt.hist(np.argmax(y_test_ohe,axis=-1),histtype='bar', ec='black',orientation="horizontal")
plt.yticks(list(label.keys()),list(label.values()))
plt.title(' Number of images for Testing')
plt.show()
In [13]:
plt.hist(np.argmax(Y_val,axis=-1),histtype='bar', ec='black',orientation="horizontal")
plt.yticks(list(label.keys()),list(label.values()))
plt.title(' Number of images for Validation')
plt.show()
In [14]:
plt.hist(np.argmax(Y_train,axis=-1),histtype='bar', ec='black',orientation="horizontal")
plt.yticks(list(label.keys()),list(label.values()))
plt.title(' Number of images for Training')
plt.show()

Normalization and Data Augmentation


Here all the dataset are normalized then, data augmentation module is fitted giving some rotation, shifts, zoom, etc. 
In [15]:
X_train,X_val =X_train/255.0 , X_val/255.0 
x_test = x_test/255.0
NUM_CLASSES = 10
In [16]:
datagen_train = ImageDataGenerator(
    featurewise_center=False,
    samplewise_center=False,
    featurewise_std_normalization=False,
    samplewise_std_normalization=False,
    zca_whitening=False,
    rotation_range=180, # used, Int. Degree range for random rotations.
    width_shift_range=0.2, # used, Float (fraction of total width). Range for random horizontal shifts.
    height_shift_range=0.2, # used,  Float (fraction of total height). Range for random vertical shifts.
    shear_range=0.1, # Float. Shear Intensity (Shear angle in counter-clockwise direction as radians)
    zoom_range=0.4,
    channel_shift_range=0.,
    fill_mode='nearest',
    cval=0.,
#     brightness_range=[0.2,1],
    horizontal_flip=True,
    vertical_flip=True,
    rescale=None)


datagen_val = ImageDataGenerator(
    featurewise_center=False,
    samplewise_center=False,
    featurewise_std_normalization=False,
    samplewise_std_normalization=False,
    zca_whitening=False,
    rotation_range=180, # used, Int. Degree range for random rotations.
    width_shift_range=0.2, # used, Float (fraction of total width). Range for random horizontal shifts.
    height_shift_range=0.2, # used,  Float (fraction of total height). Range for random vertical shifts.
    shear_range=0.1, # Float. Shear Intensity (Shear angle in counter-clockwise direction as radians)
    zoom_range=0.4,
    channel_shift_range=0.,
    fill_mode='nearest',
    cval=0.,
#     brightness_range=[0.2,1],
    horizontal_flip=True,
    vertical_flip=True,
    rescale=None)

img_wh = 128
datagen_train.fit(X_train)
datagen_val.fit(X_val)

ploting augmentated images

In [17]:
tmps = datagen_train.flow(X_train, Y_train, batch_size=3)


for i,tmp_ in enumerate(tmps):
#     imshow(tmp[i].squeeze(),cmap='bone')
    
    for j,tmp in enumerate(tmp_):
        if j == 0:
            for k,t in enumerate(tmp):
                plt.subplot(1,3,k+1)
                plt.imshow(t)
                plt.grid()
        else:
            plt.title(label[np.argmax(tmp)])
    if i ==10:  break
    plt.show()    

Model Buliding

Let's copy and build the example CNN architectures given in our class notebook.


Simple CNN

First, a very simple CNN with only two Convolution layer on top with first layer with 32 filters and kernel of 3x3 and second layer with 64 filters and kernel of 3x3. Then, a max pool layer with kernal size of 2x2,  25% Drop out, a flatten layer, 128 node dense layer and 50% drop out layer and finally a softmax output layer. 


I was having trouble plotting the models in this environment so instead I have the screenshots.

In [97]:
import matplotlib.pyplot as plt
import matplotlib.image as mpimg

fig = plt.figure(figsize=(60,2))
img = mpimg.imread('cnn.png')
plt.imshow(img)

plt.show()
In [34]:
%%time 

cnn = Sequential()

# let's start with an AlexNet style convolutional phase
cnn.add(Conv2D(filters=32,
                input_shape = (img_wh,img_wh,3),
                kernel_size=(3,3), 
                padding='same', 
                activation='relu', data_format="channels_last")) # more compact syntax

# no max pool before next conv layer!!
cnn.add(Conv2D(filters=64,
                kernel_size=(3,3), 
                padding='same', 
                activation='relu')) # more compact syntax
cnn.add(MaxPooling2D(pool_size=(2, 2), data_format="channels_last"))
    

# add one layer on flattened output
cnn.add(Dropout(0.25)) # add some dropout for regularization after conv layers
cnn.add(Flatten())
cnn.add(Dense(128, activation='relu'))
cnn.add(Dropout(0.5)) # add some dropout for regularization, again!
cnn.add(Dense(NUM_CLASSES, activation='softmax'))

# Let's train the model 
cnn.compile(loss='categorical_crossentropy', # 'categorical_crossentropy' 'mean_squared_error'
              optimizer='rmsprop', # 'adadelta' 'rmsprop'
              metrics=['accuracy'])
cnn.summary()
# the flow method yields batches of images indefinitely, with the given transformations
Model: "sequential_3"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
conv2d_19 (Conv2D)           (None, 128, 128, 32)      896       
_________________________________________________________________
conv2d_20 (Conv2D)           (None, 128, 128, 64)      18496     
_________________________________________________________________
max_pooling2d_10 (MaxPooling (None, 64, 64, 64)        0         
_________________________________________________________________
dropout_8 (Dropout)          (None, 64, 64, 64)        0         
_________________________________________________________________
flatten_4 (Flatten)          (None, 262144)            0         
_________________________________________________________________
dense_8 (Dense)              (None, 128)               33554560  
_________________________________________________________________
dropout_9 (Dropout)          (None, 128)               0         
_________________________________________________________________
dense_9 (Dense)              (None, 10)                1290      
=================================================================
Total params: 33,575,242
Trainable params: 33,575,242
Non-trainable params: 0
_________________________________________________________________
CPU times: user 76.3 ms, sys: 2.94 ms, total: 79.2 ms
Wall time: 76.6 ms
In [35]:
BS  = 32
epochs = 10
history = cnn.fit_generator(datagen_train.flow(X_train, Y_train, batch_size=BS), 
                   steps_per_epoch=len(X_train)//BS, # how many generators to go through per epoch
                   epochs=epochs, verbose=1,
                   validation_data=datagen_val.flow(X_val,Y_val, batch_size=BS),
                   validation_steps=len(X_val)//BS,
                   callbacks=[EarlyStopping(monitor='val_loss', patience=3)],
                  )
/users/sbataju/.conda/envs/tf_gpu/lib/python3.9/site-packages/keras/engine/training.py:1972: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
  warnings.warn('`Model.fit_generator` is deprecated and '
Epoch 1/10
281/281 [==============================] - 51s 180ms/step - loss: 2.4764 - accuracy: 0.2548 - val_loss: 1.6344 - val_accuracy: 0.4677
Epoch 2/10
281/281 [==============================] - 50s 179ms/step - loss: 1.7160 - accuracy: 0.4262 - val_loss: 1.3283 - val_accuracy: 0.5796
Epoch 3/10
281/281 [==============================] - 50s 179ms/step - loss: 1.4787 - accuracy: 0.5030 - val_loss: 0.9829 - val_accuracy: 0.6623
Epoch 4/10
281/281 [==============================] - 50s 178ms/step - loss: 1.3341 - accuracy: 0.5477 - val_loss: 1.5251 - val_accuracy: 0.4808
Epoch 5/10
281/281 [==============================] - 51s 180ms/step - loss: 1.2576 - accuracy: 0.5917 - val_loss: 0.9231 - val_accuracy: 0.6925
Epoch 6/10
281/281 [==============================] - 50s 176ms/step - loss: 1.1629 - accuracy: 0.6153 - val_loss: 0.7749 - val_accuracy: 0.7470
Epoch 7/10
281/281 [==============================] - 50s 179ms/step - loss: 1.1436 - accuracy: 0.6264 - val_loss: 0.8380 - val_accuracy: 0.7258
Epoch 8/10
281/281 [==============================] - 50s 178ms/step - loss: 1.1044 - accuracy: 0.6396 - val_loss: 0.7877 - val_accuracy: 0.7208
Epoch 9/10
281/281 [==============================] - 50s 177ms/step - loss: 1.0482 - accuracy: 0.6607 - val_loss: 0.6784 - val_accuracy: 0.7812
Epoch 10/10
281/281 [==============================] - 50s 177ms/step - loss: 1.0553 - accuracy: 0.6606 - val_loss: 0.6066 - val_accuracy: 0.8185

For the first try not bad at all, the accuracy is decent there is room for improvement and at least no vanishing gradiant, validation set has higher accuracy indicating that our model is underfitting and we should add more complexity to the model. We can see the confusion matrix for unseen test data is preforming similarly in terms of accuracy. And training loss is trending down. When looking at the Confusion Matrix we can read the number as the percentage as there are 100 test events

In [36]:
plot_history(history)
In [37]:
yhat = np.argmax(cnn.predict(x_test), axis=1)
acc = mt.accuracy_score(y_test,yhat)
cm = mt.confusion_matrix(y_test,yhat)
# cm =cm/sum(cm[0])
plot_confusion_matrix(cm,list(label.values()),title=f'CM Test Acc:{acc:0.3f}')
In [38]:
yhat = np.argmax(cnn.predict(X_train), axis=1)
acc = mt.accuracy_score(np.argmax(Y_train,axis=-1),yhat)
cm = mt.confusion_matrix(np.argmax(Y_train,axis=-1),yhat)
# cm =cm/sum(cm[0])
plot_confusion_matrix(cm,list(label.values()),title=f'CM Train Acc:{acc:0.3f}')
In [39]:
yhat = np.argmax(cnn.predict(X_val), axis=1)
acc = mt.accuracy_score(np.argmax(Y_val,axis=-1),yhat)
cm = mt.confusion_matrix(np.argmax(Y_val,axis=-1),yhat)
# cm =cm/sum(cm[0])
plot_confusion_matrix(cm,list(label.values()),title=f'CM Val Acc: {acc:0.3f}')

Metric

The number of images in all the classes are equal so using accuracy as metric is performing nicely. The goal to preform multiclass classification is also achived as seen in the confusion matrix above. Using accuracy mertic will insure that the opmization will maximize the diagonal element in the confusion matrix.


Simple 6 layer CNN

Now, let's try a bit complex network: two initial convolution layer with 32 filters, kernal of 3x3 then a max pool layer of 2x2 then two convolution layer with 64 filters and kernal of 3x3 and a max pool layer of 2x2 then two convolution layer of 128 filters then a max pool, flatten, dropout in bewteen a dense layer and finally a output layer.


In [98]:
fig = plt.figure(figsize=(60,2))
img = mpimg.imread('cnn11.png')
plt.imshow(img)

plt.show()
In [40]:
from tensorflow.keras.callbacks import EarlyStopping
from tensorflow.keras.regularizers import l2 
l2_lambda = 0.0001

# Use Kaiming He to regularize ReLU layers: https://arxiv.org/pdf/1502.01852.pdf
# Use Glorot/Bengio for linear/sigmoid/softmax: http://proceedings.mlr.press/v9/glorot10a/glorot10a.pdf 
cnn1 = Sequential()

cnn1.add(Conv2D(filters=32,
               input_shape = (img_wh,img_wh,3),
               kernel_size=(3,3),
               kernel_initializer='he_uniform', 
               kernel_regularizer=l2(l2_lambda),
               padding='same', 
               activation='relu',
               data_format="channels_last")) # more compact syntax

cnn1.add(Conv2D(filters=32,
               kernel_size=(3,3),
               kernel_initializer='he_uniform', 
               kernel_regularizer=l2(l2_lambda),
               padding='same', 
               activation='relu',data_format="channels_last"))
cnn1.add(MaxPooling2D(pool_size=(2, 2), data_format="channels_last"))

cnn1.add(Conv2D(filters=64,
               input_shape = (img_wh,img_wh,3),
               kernel_size=(3,3),
               kernel_initializer='he_uniform', 
               kernel_regularizer=l2(l2_lambda),
               padding='same', 
               activation='relu',data_format="channels_last")) # more compact syntax

cnn1.add(Conv2D(filters=64,
               kernel_size=(3,3),
               kernel_initializer='he_uniform', 
               kernel_regularizer=l2(l2_lambda),
               padding='same', 
               activation='relu'))
cnn1.add(MaxPooling2D(pool_size=(2, 2), data_format="channels_last"))

cnn1.add(Conv2D(filters=128,
               input_shape = (img_wh,img_wh,3),
               kernel_size=(3,3),
               kernel_initializer='he_uniform', 
               kernel_regularizer=l2(l2_lambda),
               padding='same', 
               activation='relu',data_format="channels_last")) # more compact syntax

cnn1.add(Conv2D(filters=128,
               kernel_size=(3,3),
               kernel_initializer='he_uniform', 
               kernel_regularizer=l2(l2_lambda),
               padding='same', 
               activation='relu',data_format="channels_last"))
    
cnn1.add(MaxPooling2D(pool_size=(2, 2), data_format="channels_last"))

# add one layer on flattened output

cnn1.add(Flatten())
cnn1.add(Dropout(0.25)) # add some dropout for regularization after conv layers
cnn1.add(Dense(128, 
              activation='relu',
              kernel_initializer='he_uniform',
              kernel_regularizer=l2(l2_lambda)
       ))
cnn1.add(Dropout(0.5)) # add some dropout for regularization, again!
cnn1.add(Dense(NUM_CLASSES, 
              activation='softmax', 
              kernel_initializer='glorot_uniform',
              kernel_regularizer=l2(l2_lambda)
             ))

# Let's train the model 
cnn1.compile(loss='categorical_crossentropy', # 'categorical_crossentropy' 'mean_squared_error'
              optimizer='rmsprop', # 'adadelta' 'rmsprop'
              metrics=['accuracy'])

cnn1.summary()
Model: "sequential_4"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
conv2d_21 (Conv2D)           (None, 128, 128, 32)      896       
_________________________________________________________________
conv2d_22 (Conv2D)           (None, 128, 128, 32)      9248      
_________________________________________________________________
max_pooling2d_11 (MaxPooling (None, 64, 64, 32)        0         
_________________________________________________________________
conv2d_23 (Conv2D)           (None, 64, 64, 64)        18496     
_________________________________________________________________
conv2d_24 (Conv2D)           (None, 64, 64, 64)        36928     
_________________________________________________________________
max_pooling2d_12 (MaxPooling (None, 32, 32, 64)        0         
_________________________________________________________________
conv2d_25 (Conv2D)           (None, 32, 32, 128)       73856     
_________________________________________________________________
conv2d_26 (Conv2D)           (None, 32, 32, 128)       147584    
_________________________________________________________________
max_pooling2d_13 (MaxPooling (None, 16, 16, 128)       0         
_________________________________________________________________
flatten_5 (Flatten)          (None, 32768)             0         
_________________________________________________________________
dropout_10 (Dropout)         (None, 32768)             0         
_________________________________________________________________
dense_10 (Dense)             (None, 128)               4194432   
_________________________________________________________________
dropout_11 (Dropout)         (None, 128)               0         
_________________________________________________________________
dense_11 (Dense)             (None, 10)                1290      
=================================================================
Total params: 4,482,730
Trainable params: 4,482,730
Non-trainable params: 0
_________________________________________________________________
In [41]:
BS  = 32
epochs = 10
history = cnn1.fit_generator(datagen_train.flow(X_train, Y_train, batch_size=BS), 
                   steps_per_epoch=len(X_train)//BS, # how many generators to go through per epoch
                   epochs=epochs, verbose=1,
                   validation_data=datagen_val.flow(X_val,Y_val, batch_size=BS),
                   validation_steps=len(X_val)//BS,
                   callbacks=[EarlyStopping(monitor='val_loss', patience=4)],
                  )
/users/sbataju/.conda/envs/tf_gpu/lib/python3.9/site-packages/keras/engine/training.py:1972: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
  warnings.warn('`Model.fit_generator` is deprecated and '
Epoch 1/10
281/281 [==============================] - 52s 179ms/step - loss: 2.3569 - accuracy: 0.1705 - val_loss: 1.8677 - val_accuracy: 0.3730
Epoch 2/10
281/281 [==============================] - 50s 178ms/step - loss: 1.8577 - accuracy: 0.3673 - val_loss: 1.3656 - val_accuracy: 0.5373
Epoch 3/10
281/281 [==============================] - 50s 179ms/step - loss: 1.5122 - accuracy: 0.4999 - val_loss: 1.6245 - val_accuracy: 0.4456
Epoch 4/10
281/281 [==============================] - 50s 179ms/step - loss: 1.2720 - accuracy: 0.5874 - val_loss: 0.9462 - val_accuracy: 0.7046
Epoch 5/10
281/281 [==============================] - 51s 181ms/step - loss: 1.1511 - accuracy: 0.6376 - val_loss: 0.8537 - val_accuracy: 0.7440
Epoch 6/10
281/281 [==============================] - 51s 180ms/step - loss: 1.0577 - accuracy: 0.6735 - val_loss: 0.8501 - val_accuracy: 0.7218
Epoch 7/10
281/281 [==============================] - 50s 178ms/step - loss: 1.0077 - accuracy: 0.6949 - val_loss: 0.6962 - val_accuracy: 0.7823
Epoch 8/10
281/281 [==============================] - 50s 178ms/step - loss: 0.9757 - accuracy: 0.7129 - val_loss: 0.7393 - val_accuracy: 0.7571
Epoch 9/10
281/281 [==============================] - 50s 177ms/step - loss: 0.9372 - accuracy: 0.7215 - val_loss: 0.6557 - val_accuracy: 0.8095
Epoch 10/10
281/281 [==============================] - 50s 179ms/step - loss: 0.8754 - accuracy: 0.7450 - val_loss: 0.6206 - val_accuracy: 0.8286
In [42]:
plot_history(history)
yhat = np.argmax(cnn1.predict(x_test), axis=1)
acc = mt.accuracy_score(y_test,yhat)
cm = mt.confusion_matrix(y_test,yhat)
plot_confusion_matrix(cm,list(label.values()),title= f"CM Test Acc:{acc}")
In [43]:
yhat = np.argmax(cnn1.predict(X_train), axis=1)
acc = mt.accuracy_score(np.argmax(Y_train,axis=-1),yhat)
cm = mt.confusion_matrix(np.argmax(Y_train,axis=-1),yhat)
# cm =cm/sum(cm[0])
plot_confusion_matrix(cm,list(label.values()),title=f'CM Train Acc:{acc:0.3f}')
In [44]:
yhat = np.argmax(cnn1.predict(X_val), axis=1)
acc = mt.accuracy_score(np.argmax(Y_val,axis=-1),yhat)
cm = mt.confusion_matrix(np.argmax(Y_val,axis=-1),yhat)
# cm =cm/sum(cm[0])
plot_confusion_matrix(cm,list(label.values()),title=f'CM Val Acc: {acc:0.3f}')
There is almost 4% jump in the accuracy overall than the simple CNN. From the Confusion Matrix we can see only couple of class are being misclassified for example Late blight being predicted as Early blight. And the accruacy is same accross all the dataset, the difference we see in the epoch vs accuracy plot and in the Confusion Matrix could be because the epoch vs accuracy is made when using the agumented images where as the Confusion Matrix is using the original images.


Ensemble CNN

This network is a combination of the above simple 6 layer CNN. After the two initial layer I have there branches with 32 and 64 filers convolution sandwhiched between maxpool layer then all the output is concatenate and output perdiction is made.



In [100]:
fig = plt.figure(figsize=(60,6))
img = mpimg.imread('cnn_ens1.png')
plt.imshow(img)

plt.show()
In [208]:
%%time

from tensorflow.keras.layers import Input, average, concatenate
from tensorflow.keras.models import Model

num_ensembles = 3
l2_lambda = 0.000001

input_holder = Input(shape=(img_wh, img_wh, 3))

# start with a conv layer
x = Conv2D(filters=32,
               input_shape = (img_wh,img_wh,1),
               kernel_size=(3,3),
               kernel_initializer='he_uniform', 
               kernel_regularizer=l2(l2_lambda),
               padding='same', 
               activation='relu', data_format="channels_last")(input_holder)

x = Conv2D(filters=32,
               kernel_size=(3,3),
               kernel_initializer='he_uniform', 
               kernel_regularizer=l2(l2_lambda),
               padding='same', 
               activation='relu')(x)
input_conv = MaxPooling2D(pool_size=(2, 2), data_format="channels_last")(x)

branches = []
for _ in range(num_ensembles):
    
    # start using NiN (MLPConv)
    x = Conv2D(filters=32,
                   input_shape = (img_wh,img_wh,1),
                   kernel_size=(3,3),
                   kernel_initializer='he_uniform', 
                   kernel_regularizer=l2(l2_lambda),
                   padding='same', 
                   activation='linear', data_format="channels_last")(input_conv)

    x = Conv2D(filters=32,
                   kernel_size=(1,1),
                   kernel_initializer='he_uniform', 
                   kernel_regularizer=l2(l2_lambda),
                   padding='same', 
                   activation='relu', data_format="channels_last")(x)
    
    x = MaxPooling2D(pool_size=(2, 2), data_format="channels_last")(x)
    
    x = Conv2D(filters=64,
                   input_shape = (img_wh,img_wh,1),
                   kernel_size=(3,3),
                   kernel_initializer='he_uniform', 
                   kernel_regularizer=l2(l2_lambda),
                   padding='same', 
                   activation='linear', data_format="channels_last")(x)

    x = Conv2D(filters=64,
                   kernel_size=(1,1),
                   kernel_initializer='he_uniform', 
                   kernel_regularizer=l2(l2_lambda),
                   padding='same', 
                   activation='linear', data_format="channels_last")(x)
    
    x = MaxPooling2D(pool_size=(2, 2), data_format="channels_last")(x)

    # add one layer on flattened output
    x = Flatten()(x)
    x = Dropout(0.50)(x) # add some dropout for regularization after conv layers
    x = Dense(64, 
              activation='relu',
              kernel_initializer='he_uniform',
              kernel_regularizer=l2(l2_lambda)
            )(x)
    
    x = Dense(NUM_CLASSES, 
              activation='relu',
              kernel_initializer='he_uniform',
              kernel_regularizer=l2(l2_lambda)
             )(x)
    
    # now add this branch onto the master list
    branches.append(x)

# that's it, we just need to average the results
x = concatenate(branches)

x = Dense(NUM_CLASSES, 
          activation='softmax', 
          kernel_initializer='glorot_uniform',
          kernel_regularizer=l2(l2_lambda)
         )(x)

# here is the secret sauce for setting the network using the 
#   Functional API:
cnn_ens = Model(inputs=input_holder,outputs=x)

cnn_ens.summary()

cnn_ens.compile(loss='categorical_crossentropy', # 'categorical_crossentropy' 'mean_squared_error'
                optimizer='adam', # 'adadelta' 'rmsprop'
                metrics=['accuracy'])
Model: "model_4"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_5 (InputLayer)            [(None, 128, 128, 3) 0                                            
__________________________________________________________________________________________________
conv2d_55 (Conv2D)              (None, 128, 128, 32) 896         input_5[0][0]                    
__________________________________________________________________________________________________
conv2d_56 (Conv2D)              (None, 128, 128, 32) 9248        conv2d_55[0][0]                  
__________________________________________________________________________________________________
max_pooling2d_28 (MaxPooling2D) (None, 64, 64, 32)   0           conv2d_56[0][0]                  
__________________________________________________________________________________________________
conv2d_57 (Conv2D)              (None, 64, 64, 32)   9248        max_pooling2d_28[0][0]           
__________________________________________________________________________________________________
conv2d_61 (Conv2D)              (None, 64, 64, 32)   9248        max_pooling2d_28[0][0]           
__________________________________________________________________________________________________
conv2d_65 (Conv2D)              (None, 64, 64, 32)   9248        max_pooling2d_28[0][0]           
__________________________________________________________________________________________________
conv2d_58 (Conv2D)              (None, 64, 64, 32)   1056        conv2d_57[0][0]                  
__________________________________________________________________________________________________
conv2d_62 (Conv2D)              (None, 64, 64, 32)   1056        conv2d_61[0][0]                  
__________________________________________________________________________________________________
conv2d_66 (Conv2D)              (None, 64, 64, 32)   1056        conv2d_65[0][0]                  
__________________________________________________________________________________________________
max_pooling2d_29 (MaxPooling2D) (None, 32, 32, 32)   0           conv2d_58[0][0]                  
__________________________________________________________________________________________________
max_pooling2d_31 (MaxPooling2D) (None, 32, 32, 32)   0           conv2d_62[0][0]                  
__________________________________________________________________________________________________
max_pooling2d_33 (MaxPooling2D) (None, 32, 32, 32)   0           conv2d_66[0][0]                  
__________________________________________________________________________________________________
conv2d_59 (Conv2D)              (None, 32, 32, 64)   18496       max_pooling2d_29[0][0]           
__________________________________________________________________________________________________
conv2d_63 (Conv2D)              (None, 32, 32, 64)   18496       max_pooling2d_31[0][0]           
__________________________________________________________________________________________________
conv2d_67 (Conv2D)              (None, 32, 32, 64)   18496       max_pooling2d_33[0][0]           
__________________________________________________________________________________________________
conv2d_60 (Conv2D)              (None, 32, 32, 64)   4160        conv2d_59[0][0]                  
__________________________________________________________________________________________________
conv2d_64 (Conv2D)              (None, 32, 32, 64)   4160        conv2d_63[0][0]                  
__________________________________________________________________________________________________
conv2d_68 (Conv2D)              (None, 32, 32, 64)   4160        conv2d_67[0][0]                  
__________________________________________________________________________________________________
max_pooling2d_30 (MaxPooling2D) (None, 16, 16, 64)   0           conv2d_60[0][0]                  
__________________________________________________________________________________________________
max_pooling2d_32 (MaxPooling2D) (None, 16, 16, 64)   0           conv2d_64[0][0]                  
__________________________________________________________________________________________________
max_pooling2d_34 (MaxPooling2D) (None, 16, 16, 64)   0           conv2d_68[0][0]                  
__________________________________________________________________________________________________
flatten_14 (Flatten)            (None, 16384)        0           max_pooling2d_30[0][0]           
__________________________________________________________________________________________________
flatten_15 (Flatten)            (None, 16384)        0           max_pooling2d_32[0][0]           
__________________________________________________________________________________________________
flatten_16 (Flatten)            (None, 16384)        0           max_pooling2d_34[0][0]           
__________________________________________________________________________________________________
dropout_22 (Dropout)            (None, 16384)        0           flatten_14[0][0]                 
__________________________________________________________________________________________________
dropout_23 (Dropout)            (None, 16384)        0           flatten_15[0][0]                 
__________________________________________________________________________________________________
dropout_24 (Dropout)            (None, 16384)        0           flatten_16[0][0]                 
__________________________________________________________________________________________________
dense_30 (Dense)                (None, 64)           1048640     dropout_22[0][0]                 
__________________________________________________________________________________________________
dense_32 (Dense)                (None, 64)           1048640     dropout_23[0][0]                 
__________________________________________________________________________________________________
dense_34 (Dense)                (None, 64)           1048640     dropout_24[0][0]                 
__________________________________________________________________________________________________
dense_31 (Dense)                (None, 10)           650         dense_30[0][0]                   
__________________________________________________________________________________________________
dense_33 (Dense)                (None, 10)           650         dense_32[0][0]                   
__________________________________________________________________________________________________
dense_35 (Dense)                (None, 10)           650         dense_34[0][0]                   
__________________________________________________________________________________________________
concatenate_2 (Concatenate)     (None, 30)           0           dense_31[0][0]                   
                                                                 dense_33[0][0]                   
                                                                 dense_35[0][0]                   
__________________________________________________________________________________________________
dense_36 (Dense)                (None, 10)           310         concatenate_2[0][0]              
==================================================================================================
Total params: 3,257,204
Trainable params: 3,257,204
Non-trainable params: 0
__________________________________________________________________________________________________
CPU times: user 219 ms, sys: 8.07 ms, total: 227 ms
Wall time: 224 ms
In [209]:
BS  = 32
epochs = 10
history = cnn_ens.fit_generator(datagen_train.flow(X_train, Y_train, batch_size=BS), 
                   steps_per_epoch=len(X_train)//BS, # how many generators to go through per epoch
                   epochs=epochs, verbose=1,
                   validation_data=datagen_val.flow(X_val,Y_val, batch_size=BS),
                   validation_steps=len(X_val)//BS,
                   callbacks=[EarlyStopping(monitor='val_loss', patience=4)],
                  )
/users/sbataju/.conda/envs/tf_gpu/lib/python3.9/site-packages/keras/engine/training.py:1972: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
  warnings.warn('`Model.fit_generator` is deprecated and '
Epoch 1/10
281/281 [==============================] - 46s 157ms/step - loss: 2.4052 - accuracy: 0.0960 - val_loss: 2.3044 - val_accuracy: 0.1008
Epoch 2/10
281/281 [==============================] - 44s 156ms/step - loss: 2.3046 - accuracy: 0.0938 - val_loss: 2.3044 - val_accuracy: 0.1008
Epoch 3/10
281/281 [==============================] - 43s 155ms/step - loss: 2.3046 - accuracy: 0.0938 - val_loss: 2.3044 - val_accuracy: 0.0998
Epoch 4/10
281/281 [==============================] - 44s 155ms/step - loss: 2.3046 - accuracy: 0.0972 - val_loss: 2.3044 - val_accuracy: 0.0998
Epoch 5/10
281/281 [==============================] - 44s 157ms/step - loss: 2.3046 - accuracy: 0.0969 - val_loss: 2.3044 - val_accuracy: 0.1008
Epoch 6/10
281/281 [==============================] - 44s 155ms/step - loss: 2.3046 - accuracy: 0.0960 - val_loss: 2.3044 - val_accuracy: 0.0998
Epoch 7/10
281/281 [==============================] - 44s 156ms/step - loss: 2.3046 - accuracy: 0.0929 - val_loss: 2.3044 - val_accuracy: 0.0998
Epoch 8/10
281/281 [==============================] - 43s 154ms/step - loss: 2.3046 - accuracy: 0.0944 - val_loss: 2.3043 - val_accuracy: 0.1008
Epoch 9/10
281/281 [==============================] - 43s 154ms/step - loss: 2.3046 - accuracy: 0.0902 - val_loss: 2.3043 - val_accuracy: 0.1008
Epoch 10/10
281/281 [==============================] - 44s 155ms/step - loss: 2.3046 - accuracy: 0.0933 - val_loss: 2.3043 - val_accuracy: 0.1008
In [210]:
plot_history(history)
yhat = np.argmax(cnn_ens.predict(x_test), axis=1)
acc = mt.accuracy_score(y_test,yhat)
cm = mt.confusion_matrix(y_test,yhat)
plot_confusion_matrix(cm,list(label.values()),title= f"CM Test Acc:{acc}")
Either the default hyperparameter is not working or the architecture is not suitable, either way I will leave it and move on to other techniques.


Resnet type CNN

A Resnet style connection for middle convolution layers.


In [102]:
fig = plt.figure(figsize=(60,6))
img = mpimg.imread('resnet1.png')
plt.imshow(img)
plt.show()
In [45]:
%%time
 
from tensorflow.keras.layers import Add, Input
from tensorflow.keras.layers import average, concatenate
from tensorflow.keras.models import Model

input_holder = Input(shape=(img_wh, img_wh, 3))

# start with a conv layer
x = Conv2D(filters=32,
               input_shape = (img_wh,img_wh,1),
               kernel_size=(3,3),
               kernel_initializer='he_uniform', 
               kernel_regularizer=l2(l2_lambda),
               padding='same', 
               activation='relu', 
               data_format="channels_last")(input_holder)

x = MaxPooling2D(pool_size=(2, 2), data_format="channels_last")(x)

x = Conv2D(filters=32,
               kernel_size=(3,3),
               kernel_initializer='he_uniform', 
               kernel_regularizer=l2(l2_lambda),
               padding='same', 
               activation='relu', 
               data_format="channels_last")(x)

x_split = MaxPooling2D(pool_size=(2, 2), data_format="channels_last")(x)

x = Conv2D(filters=64,
               kernel_size=(1,1),
               kernel_initializer='he_uniform', 
               kernel_regularizer=l2(l2_lambda),
               padding='same', 
               activation='relu', 
               data_format="channels_last")(x_split)

x = Conv2D(filters=64,
               kernel_size=(3,3),
               kernel_initializer='he_uniform', 
               kernel_regularizer=l2(l2_lambda),
               padding='same', 
               activation='relu', 
               data_format="channels_last")(x)

x = Conv2D(filters=32,
               kernel_size=(1,1),
               kernel_initializer='he_uniform', 
               kernel_regularizer=l2(l2_lambda),
               padding='same', 
               activation='relu', 
               data_format="channels_last")(x)

# now add back in the split layer, x_split (residual added in)
x = Add()([x, x_split])
x = Activation("relu")(x)

x = MaxPooling2D(pool_size=(2, 2), data_format="channels_last")(x)

x = Flatten()(x)
x = Dropout(0.25)(x)
x = Dense(256)(x)
x = Activation("relu")(x)
x = Dropout(0.5)(x)
x = Dense(NUM_CLASSES)(x)
x = Activation('softmax')(x)

resnet = Model(inputs=input_holder,outputs=x)

resnet.summary()
Model: "model_1"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_2 (InputLayer)            [(None, 128, 128, 3) 0                                            
__________________________________________________________________________________________________
conv2d_27 (Conv2D)              (None, 128, 128, 32) 896         input_2[0][0]                    
__________________________________________________________________________________________________
max_pooling2d_14 (MaxPooling2D) (None, 64, 64, 32)   0           conv2d_27[0][0]                  
__________________________________________________________________________________________________
conv2d_28 (Conv2D)              (None, 64, 64, 32)   9248        max_pooling2d_14[0][0]           
__________________________________________________________________________________________________
max_pooling2d_15 (MaxPooling2D) (None, 32, 32, 32)   0           conv2d_28[0][0]                  
__________________________________________________________________________________________________
conv2d_29 (Conv2D)              (None, 32, 32, 64)   2112        max_pooling2d_15[0][0]           
__________________________________________________________________________________________________
conv2d_30 (Conv2D)              (None, 32, 32, 64)   36928       conv2d_29[0][0]                  
__________________________________________________________________________________________________
conv2d_31 (Conv2D)              (None, 32, 32, 32)   2080        conv2d_30[0][0]                  
__________________________________________________________________________________________________
add_1 (Add)                     (None, 32, 32, 32)   0           conv2d_31[0][0]                  
                                                                 max_pooling2d_15[0][0]           
__________________________________________________________________________________________________
activation_3 (Activation)       (None, 32, 32, 32)   0           add_1[0][0]                      
__________________________________________________________________________________________________
max_pooling2d_16 (MaxPooling2D) (None, 16, 16, 32)   0           activation_3[0][0]               
__________________________________________________________________________________________________
flatten_6 (Flatten)             (None, 8192)         0           max_pooling2d_16[0][0]           
__________________________________________________________________________________________________
dropout_12 (Dropout)            (None, 8192)         0           flatten_6[0][0]                  
__________________________________________________________________________________________________
dense_12 (Dense)                (None, 256)          2097408     dropout_12[0][0]                 
__________________________________________________________________________________________________
activation_4 (Activation)       (None, 256)          0           dense_12[0][0]                   
__________________________________________________________________________________________________
dropout_13 (Dropout)            (None, 256)          0           activation_4[0][0]               
__________________________________________________________________________________________________
dense_13 (Dense)                (None, 10)           2570        dropout_13[0][0]                 
__________________________________________________________________________________________________
activation_5 (Activation)       (None, 10)           0           dense_13[0][0]                   
==================================================================================================
Total params: 2,151,242
Trainable params: 2,151,242
Non-trainable params: 0
__________________________________________________________________________________________________
CPU times: user 105 ms, sys: 3.03 ms, total: 108 ms
Wall time: 106 ms
In [46]:
resnet.compile(loss='categorical_crossentropy', # 'categorical_crossentropy' 'mean_squared_error'
                optimizer='adam', # 'adadelta' 'rmsprop'
                metrics=['accuracy'])
In [62]:
BS  = 32
epochs = 20
history = resnet.fit_generator(datagen_train.flow(X_train, Y_train, batch_size=BS), 
                   steps_per_epoch=len(X_train)//BS, # how many generators to go through per epoch
                   epochs=epochs, verbose=1,
                   validation_data=datagen_val.flow(X_val,Y_val, batch_size=BS),
                   validation_steps=len(X_val)//BS,
                   callbacks=[EarlyStopping(monitor='val_loss', patience=4)],
                  )
/users/sbataju/.conda/envs/tf_gpu/lib/python3.9/site-packages/keras/engine/training.py:1972: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
  warnings.warn('`Model.fit_generator` is deprecated and '
Epoch 1/20
281/281 [==============================] - 50s 179ms/step - loss: 0.6489 - accuracy: 0.7893 - val_loss: 0.4655 - val_accuracy: 0.8498
Epoch 2/20
281/281 [==============================] - 50s 177ms/step - loss: 0.6485 - accuracy: 0.7907 - val_loss: 0.4786 - val_accuracy: 0.8407
Epoch 3/20
281/281 [==============================] - 50s 178ms/step - loss: 0.6043 - accuracy: 0.8039 - val_loss: 0.4946 - val_accuracy: 0.8558
Epoch 4/20
281/281 [==============================] - 49s 176ms/step - loss: 0.5972 - accuracy: 0.8075 - val_loss: 0.4947 - val_accuracy: 0.8417
Epoch 5/20
281/281 [==============================] - 51s 180ms/step - loss: 0.5831 - accuracy: 0.8108 - val_loss: 0.4289 - val_accuracy: 0.8679
Epoch 6/20
281/281 [==============================] - 49s 173ms/step - loss: 0.5476 - accuracy: 0.8184 - val_loss: 0.4050 - val_accuracy: 0.8700
Epoch 7/20
281/281 [==============================] - 50s 177ms/step - loss: 0.5988 - accuracy: 0.8039 - val_loss: 0.5096 - val_accuracy: 0.8347
Epoch 8/20
281/281 [==============================] - 50s 179ms/step - loss: 0.5557 - accuracy: 0.8211 - val_loss: 0.3705 - val_accuracy: 0.8790
Epoch 9/20
281/281 [==============================] - 47s 169ms/step - loss: 0.5389 - accuracy: 0.8291 - val_loss: 0.3987 - val_accuracy: 0.8810
Epoch 10/20
281/281 [==============================] - 50s 178ms/step - loss: 0.5218 - accuracy: 0.8330 - val_loss: 0.3836 - val_accuracy: 0.8790
Epoch 11/20
281/281 [==============================] - 50s 179ms/step - loss: 0.5210 - accuracy: 0.8331 - val_loss: 0.3363 - val_accuracy: 0.8931
Epoch 12/20
281/281 [==============================] - 50s 180ms/step - loss: 0.5199 - accuracy: 0.8298 - val_loss: 0.3556 - val_accuracy: 0.8962
Epoch 13/20
281/281 [==============================] - 50s 179ms/step - loss: 0.5121 - accuracy: 0.8399 - val_loss: 0.3791 - val_accuracy: 0.8710
Epoch 14/20
281/281 [==============================] - 50s 179ms/step - loss: 0.5069 - accuracy: 0.8366 - val_loss: 0.3511 - val_accuracy: 0.9062
Epoch 15/20
281/281 [==============================] - 49s 176ms/step - loss: 0.4913 - accuracy: 0.8405 - val_loss: 0.3129 - val_accuracy: 0.9002
Epoch 16/20
281/281 [==============================] - 50s 178ms/step - loss: 0.4912 - accuracy: 0.8467 - val_loss: 0.4080 - val_accuracy: 0.8700
Epoch 17/20
281/281 [==============================] - 50s 179ms/step - loss: 0.4528 - accuracy: 0.8588 - val_loss: 0.2902 - val_accuracy: 0.9163
Epoch 18/20
281/281 [==============================] - 49s 176ms/step - loss: 0.4709 - accuracy: 0.8524 - val_loss: 0.4149 - val_accuracy: 0.8690
Epoch 19/20
281/281 [==============================] - 50s 178ms/step - loss: 0.4719 - accuracy: 0.8519 - val_loss: 0.3243 - val_accuracy: 0.8992
Epoch 20/20
281/281 [==============================] - 50s 179ms/step - loss: 0.4492 - accuracy: 0.8599 - val_loss: 0.3084 - val_accuracy: 0.9083
In [63]:
plot_history(history)
yhat = np.argmax(resnet.predict(x_test), axis=1)
acc = mt.accuracy_score(y_test,yhat)
cm = mt.confusion_matrix(y_test,yhat)
plot_confusion_matrix(cm,list(label.values()),title= f"CM Test Acc:{acc}")
In [64]:
yhat = np.argmax(resnet.predict(X_train), axis=1)
acc = mt.accuracy_score(np.argmax(Y_train,axis=-1),yhat)
cm = mt.confusion_matrix(np.argmax(Y_train,axis=-1),yhat)
# cm =cm/sum(cm[0])
plot_confusion_matrix(cm,list(label.values()),title=f'CM Train Acc:{acc:0.3f}')
In [65]:
yhat = np.argmax(resnet.predict(X_val), axis=1)
acc = mt.accuracy_score(np.argmax(Y_val,axis=-1),yhat)
cm = mt.confusion_matrix(np.argmax(Y_val,axis=-1),yhat)
# cm =cm/sum(cm[0])
plot_confusion_matrix(cm,list(label.values()),title=f'CM Val Acc: {acc:0.3f}')
Couple of percentage points off from the 6 layer CNN but still good results.


Xception style CNN

Two separable convolution layer in between.


In [103]:
fig = plt.figure(figsize=(60,6))
img = mpimg.imread('xception1.png')
plt.imshow(img)

plt.show()
In [254]:
# Xception style architecture
from tensorflow.keras.layers import SeparableConv2D
from tensorflow.keras.layers import BatchNormalization
from tensorflow.keras.layers import Add, Input
from tensorflow.keras.layers import average, concatenate
from tensorflow.keras.models import Model

l2_lambda = 0.000001



input_holder = Input(shape=(img_wh, img_wh, 3))

# start with a conv layer
x = Conv2D(filters=32,
               input_shape = (img_wh,img_wh,1),
               kernel_size=(3,3),
               kernel_initializer='he_uniform', 
               kernel_regularizer=l2(l2_lambda),
               padding='same', 
               activation='relu', 
               data_format="channels_last")(input_holder)

x = MaxPooling2D(pool_size=(2, 2), data_format="channels_last")(x)

x = Conv2D(filters=32,
               kernel_size=(3,3),
               kernel_initializer='he_uniform', 
               kernel_regularizer=l2(l2_lambda),
               padding='same', 
               activation='relu', 
               data_format="channels_last")(x)


x_split = MaxPooling2D(pool_size=(2, 2), data_format="channels_last")(x)

x = SeparableConv2D(filters=32,
               input_shape = (img_wh,img_wh,1),
               kernel_size=(3,3),
               kernel_initializer='he_uniform', 
               kernel_regularizer=l2(l2_lambda),
               padding='same', 
               activation='relu', 
               depth_multiplier = 1, # controls output channels
               data_format="channels_last")(x_split)


x_split = Add()([x, x_split])

x = SeparableConv2D(filters=32,
               input_shape = (img_wh,img_wh,1),
               kernel_size=(3,3),
               kernel_initializer='he_uniform', 
               kernel_regularizer=l2(l2_lambda),
               padding='same', 
               activation='relu', 
               depth_multiplier = 1, # controls output channels
               data_format="channels_last")(x_split)


x_split = Add()([x, x_split])


x = Activation("relu")(x_split)

x = MaxPooling2D(pool_size=(2, 2), data_format="channels_last")(x)

x = Flatten()(x)
x = Dropout(0.25)(x)
x = Dense(256, activation="relu")(x)
x = Dropout(0.5)(x)
x = Dense(NUM_CLASSES,activation="softmax")(x)

xception = Model(inputs=input_holder,outputs=x)

xception.summary()
Model: "model_12"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_14 (InputLayer)           [(None, 128, 128, 3) 0                                            
__________________________________________________________________________________________________
conv2d_85 (Conv2D)              (None, 128, 128, 32) 896         input_14[0][0]                   
__________________________________________________________________________________________________
max_pooling2d_64 (MaxPooling2D) (None, 64, 64, 32)   0           conv2d_85[0][0]                  
__________________________________________________________________________________________________
conv2d_86 (Conv2D)              (None, 64, 64, 32)   9248        max_pooling2d_64[0][0]           
__________________________________________________________________________________________________
max_pooling2d_65 (MaxPooling2D) (None, 32, 32, 32)   0           conv2d_86[0][0]                  
__________________________________________________________________________________________________
separable_conv2d_17 (SeparableC (None, 32, 32, 32)   1344        max_pooling2d_65[0][0]           
__________________________________________________________________________________________________
add_18 (Add)                    (None, 32, 32, 32)   0           separable_conv2d_17[0][0]        
                                                                 max_pooling2d_65[0][0]           
__________________________________________________________________________________________________
separable_conv2d_18 (SeparableC (None, 32, 32, 32)   1344        add_18[0][0]                     
__________________________________________________________________________________________________
add_19 (Add)                    (None, 32, 32, 32)   0           separable_conv2d_18[0][0]        
                                                                 add_18[0][0]                     
__________________________________________________________________________________________________
activation_17 (Activation)      (None, 32, 32, 32)   0           add_19[0][0]                     
__________________________________________________________________________________________________
max_pooling2d_66 (MaxPooling2D) (None, 16, 16, 32)   0           activation_17[0][0]              
__________________________________________________________________________________________________
flatten_24 (Flatten)            (None, 8192)         0           max_pooling2d_66[0][0]           
__________________________________________________________________________________________________
dropout_45 (Dropout)            (None, 8192)         0           flatten_24[0][0]                 
__________________________________________________________________________________________________
dense_57 (Dense)                (None, 256)          2097408     dropout_45[0][0]                 
__________________________________________________________________________________________________
dropout_46 (Dropout)            (None, 256)          0           dense_57[0][0]                   
__________________________________________________________________________________________________
dense_58 (Dense)                (None, 10)           2570        dropout_46[0][0]                 
==================================================================================================
Total params: 2,112,810
Trainable params: 2,112,810
Non-trainable params: 0
__________________________________________________________________________________________________
In [255]:
# speed up by training by not using augmentation, perhaps there are faster ways??
xception.compile(loss='categorical_crossentropy', # 'categorical_crossentropy' 'mean_squared_error'
                optimizer='adam', # 'adadelta' 'rmsprop'
                metrics=['accuracy'])
In [256]:
BS  = 32
epochs = 50
history = xception.fit_generator(datagen_train.flow(X_train, Y_train, batch_size=BS), 
                   steps_per_epoch=len(X_train)//BS, # how many generators to go through per epoch
                   epochs=epochs, verbose=1,
                   validation_data=datagen_val.flow(X_val,Y_val, batch_size=BS),
                   validation_steps=len(X_val)//BS,
                   callbacks=[EarlyStopping(monitor='val_loss', patience=4)],
                  )
Epoch 1/50
281/281 [==============================] - 45s 158ms/step - loss: 2.2368 - accuracy: 0.1974 - val_loss: 1.7111 - val_accuracy: 0.4446
Epoch 2/50
281/281 [==============================] - 44s 156ms/step - loss: 1.6281 - accuracy: 0.4206 - val_loss: 1.3291 - val_accuracy: 0.5534
Epoch 3/50
281/281 [==============================] - 44s 155ms/step - loss: 1.2567 - accuracy: 0.5542 - val_loss: 0.9238 - val_accuracy: 0.6875
Epoch 4/50
281/281 [==============================] - 43s 154ms/step - loss: 1.0842 - accuracy: 0.6062 - val_loss: 0.8590 - val_accuracy: 0.7097
Epoch 5/50
281/281 [==============================] - 44s 155ms/step - loss: 0.9757 - accuracy: 0.6533 - val_loss: 0.8483 - val_accuracy: 0.6825
Epoch 6/50
281/281 [==============================] - 43s 154ms/step - loss: 0.9101 - accuracy: 0.6785 - val_loss: 0.7571 - val_accuracy: 0.7228
Epoch 7/50
281/281 [==============================] - 43s 154ms/step - loss: 0.8389 - accuracy: 0.7019 - val_loss: 0.6063 - val_accuracy: 0.8004
Epoch 8/50
281/281 [==============================] - 44s 156ms/step - loss: 0.7985 - accuracy: 0.7240 - val_loss: 0.5365 - val_accuracy: 0.8306
Epoch 9/50
281/281 [==============================] - 43s 154ms/step - loss: 0.7427 - accuracy: 0.7422 - val_loss: 0.4914 - val_accuracy: 0.8327
Epoch 10/50
281/281 [==============================] - 43s 154ms/step - loss: 0.7204 - accuracy: 0.7463 - val_loss: 0.4410 - val_accuracy: 0.8508
Epoch 11/50
281/281 [==============================] - 43s 153ms/step - loss: 0.7191 - accuracy: 0.7510 - val_loss: 0.4993 - val_accuracy: 0.8377
Epoch 12/50
281/281 [==============================] - 43s 153ms/step - loss: 0.6672 - accuracy: 0.7717 - val_loss: 0.5063 - val_accuracy: 0.8397
Epoch 13/50
281/281 [==============================] - 43s 154ms/step - loss: 0.6621 - accuracy: 0.7706 - val_loss: 0.4797 - val_accuracy: 0.8337
Epoch 14/50
281/281 [==============================] - 43s 154ms/step - loss: 0.6395 - accuracy: 0.7773 - val_loss: 0.4484 - val_accuracy: 0.8438
In [260]:
plot_history(history)
yhat = np.argmax(xception.predict(x_test), axis=1)
acc = mt.accuracy_score(y_test,yhat)
cm = mt.confusion_matrix(y_test,yhat)
plot_confusion_matrix(cm,list(label.values()),title= f"CM Test Acc:{acc}")
I had trained the Resnet and Xception to 50 epoch but when I reran the notebook I had change the Resnet network to 10 epoch.
Xception is also working properly but for now lets foucs on Resnet. 

Less Drop Out on Resnet type CNN

Changed the Drop out percentage from 25,50 to 85,85. 


In [267]:
%%time

# now lets use the LeNet architecture with batch norm
# We will also use ReLU where approriate and drop out 
from tensorflow.keras.layers import Add, Input
from tensorflow.keras.layers import average, concatenate
from tensorflow.keras.models import Model

input_holder = Input(shape=(img_wh, img_wh, 3))

# start with a conv layer
x = Conv2D(filters=32,
               input_shape = (img_wh,img_wh,1),
               kernel_size=(3,3),
               kernel_initializer='he_uniform', 
               kernel_regularizer=l2(l2_lambda),
               padding='same', 
               activation='relu', 
               data_format="channels_last")(input_holder)

x = MaxPooling2D(pool_size=(2, 2), data_format="channels_last")(x)

x = Conv2D(filters=32,
               kernel_size=(3,3),
               kernel_initializer='he_uniform', 
               kernel_regularizer=l2(l2_lambda),
               padding='same', 
               activation='relu', 
               data_format="channels_last")(x)

x_split = MaxPooling2D(pool_size=(2, 2), data_format="channels_last")(x)

x = Conv2D(filters=64,
               kernel_size=(1,1),
               kernel_initializer='he_uniform', 
               kernel_regularizer=l2(l2_lambda),
               padding='same', 
               activation='relu', 
               data_format="channels_last")(x_split)

x = Conv2D(filters=64,
               kernel_size=(3,3),
               kernel_initializer='he_uniform', 
               kernel_regularizer=l2(l2_lambda),
               padding='same', 
               activation='relu', 
               data_format="channels_last")(x)

x = Conv2D(filters=32,
               kernel_size=(1,1),
               kernel_initializer='he_uniform', 
               kernel_regularizer=l2(l2_lambda),
               padding='same', 
               activation='relu', 
               data_format="channels_last")(x)

# now add back in the split layer, x_split (residual added in)
x = Add()([x, x_split])
x = Activation("relu")(x)

x = MaxPooling2D(pool_size=(2, 2), data_format="channels_last")(x)

x = Flatten()(x)
x = Dropout(0.85)(x)
x = Dense(256)(x)
x = Activation("relu")(x)
x = Dropout(0.85)(x)
x = Dense(NUM_CLASSES)(x)
x = Activation('softmax')(x)

resnet1 = Model(inputs=input_holder,outputs=x)

resnet1.summary()
Model: "model_16"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_18 (InputLayer)           [(None, 128, 128, 3) 0                                            
__________________________________________________________________________________________________
conv2d_102 (Conv2D)             (None, 128, 128, 32) 896         input_18[0][0]                   
__________________________________________________________________________________________________
max_pooling2d_76 (MaxPooling2D) (None, 64, 64, 32)   0           conv2d_102[0][0]                 
__________________________________________________________________________________________________
conv2d_103 (Conv2D)             (None, 64, 64, 32)   9248        max_pooling2d_76[0][0]           
__________________________________________________________________________________________________
max_pooling2d_77 (MaxPooling2D) (None, 32, 32, 32)   0           conv2d_103[0][0]                 
__________________________________________________________________________________________________
conv2d_104 (Conv2D)             (None, 32, 32, 64)   2112        max_pooling2d_77[0][0]           
__________________________________________________________________________________________________
conv2d_105 (Conv2D)             (None, 32, 32, 64)   36928       conv2d_104[0][0]                 
__________________________________________________________________________________________________
conv2d_106 (Conv2D)             (None, 32, 32, 32)   2080        conv2d_105[0][0]                 
__________________________________________________________________________________________________
add_23 (Add)                    (None, 32, 32, 32)   0           conv2d_106[0][0]                 
                                                                 max_pooling2d_77[0][0]           
__________________________________________________________________________________________________
activation_27 (Activation)      (None, 32, 32, 32)   0           add_23[0][0]                     
__________________________________________________________________________________________________
max_pooling2d_78 (MaxPooling2D) (None, 16, 16, 32)   0           activation_27[0][0]              
__________________________________________________________________________________________________
flatten_28 (Flatten)            (None, 8192)         0           max_pooling2d_78[0][0]           
__________________________________________________________________________________________________
dropout_53 (Dropout)            (None, 8192)         0           flatten_28[0][0]                 
__________________________________________________________________________________________________
dense_65 (Dense)                (None, 256)          2097408     dropout_53[0][0]                 
__________________________________________________________________________________________________
activation_28 (Activation)      (None, 256)          0           dense_65[0][0]                   
__________________________________________________________________________________________________
dropout_54 (Dropout)            (None, 256)          0           activation_28[0][0]              
__________________________________________________________________________________________________
dense_66 (Dense)                (None, 10)           2570        dropout_54[0][0]                 
__________________________________________________________________________________________________
activation_29 (Activation)      (None, 10)           0           dense_66[0][0]                   
==================================================================================================
Total params: 2,151,242
Trainable params: 2,151,242
Non-trainable params: 0
__________________________________________________________________________________________________
CPU times: user 83.8 ms, sys: 9.9 ms, total: 93.7 ms
Wall time: 92 ms
In [268]:
resnet1.compile(loss='categorical_crossentropy', # 'categorical_crossentropy' 'mean_squared_error'
                optimizer='adam', # 'adadelta' 'rmsprop'
                metrics=['accuracy'])
In [269]:
BS  = 32
epochs = 50
history = resnet1.fit_generator(datagen_train.flow(X_train, Y_train, batch_size=BS), 
                   steps_per_epoch=len(X_train)//BS, # how many generators to go through per epoch
                   epochs=epochs, verbose=1,
                   validation_data=datagen_val.flow(X_val,Y_val, batch_size=BS),
                   validation_steps=len(X_val)//BS,
                   callbacks=[EarlyStopping(monitor='val_loss', patience=4)],
                  )
/users/sbataju/.conda/envs/tf_gpu/lib/python3.9/site-packages/keras/engine/training.py:1972: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
  warnings.warn('`Model.fit_generator` is deprecated and '
Epoch 1/50
281/281 [==============================] - 45s 157ms/step - loss: 2.4728 - accuracy: 0.1098 - val_loss: 2.2623 - val_accuracy: 0.1905
Epoch 2/50
281/281 [==============================] - 44s 155ms/step - loss: 2.2120 - accuracy: 0.1703 - val_loss: 2.1216 - val_accuracy: 0.2893
Epoch 3/50
281/281 [==============================] - 44s 155ms/step - loss: 2.1108 - accuracy: 0.2275 - val_loss: 1.8863 - val_accuracy: 0.3357
Epoch 4/50
281/281 [==============================] - 44s 155ms/step - loss: 1.9815 - accuracy: 0.3022 - val_loss: 1.7778 - val_accuracy: 0.4183
Epoch 5/50
281/281 [==============================] - 43s 154ms/step - loss: 1.8772 - accuracy: 0.3391 - val_loss: 1.5930 - val_accuracy: 0.4839
Epoch 6/50
281/281 [==============================] - 44s 155ms/step - loss: 1.7788 - accuracy: 0.3712 - val_loss: 1.5299 - val_accuracy: 0.4718
Epoch 7/50
281/281 [==============================] - 43s 154ms/step - loss: 1.7180 - accuracy: 0.3926 - val_loss: 1.5742 - val_accuracy: 0.4567
Epoch 8/50
281/281 [==============================] - 43s 154ms/step - loss: 1.6238 - accuracy: 0.4241 - val_loss: 1.3994 - val_accuracy: 0.4950
Epoch 9/50
281/281 [==============================] - 43s 154ms/step - loss: 1.5488 - accuracy: 0.4419 - val_loss: 1.2327 - val_accuracy: 0.5554
Epoch 10/50
281/281 [==============================] - 44s 156ms/step - loss: 1.4823 - accuracy: 0.4677 - val_loss: 1.2381 - val_accuracy: 0.5595
Epoch 11/50
281/281 [==============================] - 44s 155ms/step - loss: 1.4371 - accuracy: 0.4853 - val_loss: 1.1228 - val_accuracy: 0.6089
Epoch 12/50
281/281 [==============================] - 43s 153ms/step - loss: 1.3780 - accuracy: 0.5091 - val_loss: 1.1143 - val_accuracy: 0.5847
Epoch 13/50
281/281 [==============================] - 43s 154ms/step - loss: 1.3346 - accuracy: 0.5251 - val_loss: 1.0030 - val_accuracy: 0.6401
Epoch 14/50
281/281 [==============================] - 43s 154ms/step - loss: 1.2710 - accuracy: 0.5437 - val_loss: 1.0227 - val_accuracy: 0.6210
Epoch 15/50
281/281 [==============================] - 43s 154ms/step - loss: 1.2040 - accuracy: 0.5579 - val_loss: 1.0124 - val_accuracy: 0.6321
Epoch 16/50
281/281 [==============================] - 43s 154ms/step - loss: 1.2058 - accuracy: 0.5711 - val_loss: 0.9317 - val_accuracy: 0.6613
Epoch 17/50
281/281 [==============================] - 44s 155ms/step - loss: 1.1756 - accuracy: 0.5753 - val_loss: 0.8681 - val_accuracy: 0.6835
Epoch 18/50
281/281 [==============================] - 43s 154ms/step - loss: 1.1607 - accuracy: 0.5852 - val_loss: 0.8663 - val_accuracy: 0.6966
Epoch 19/50
281/281 [==============================] - 43s 153ms/step - loss: 1.1516 - accuracy: 0.5856 - val_loss: 0.8329 - val_accuracy: 0.7046
Epoch 20/50
281/281 [==============================] - 44s 155ms/step - loss: 1.1202 - accuracy: 0.6016 - val_loss: 0.8658 - val_accuracy: 0.7046
Epoch 21/50
281/281 [==============================] - 43s 155ms/step - loss: 1.1141 - accuracy: 0.6005 - val_loss: 0.8841 - val_accuracy: 0.7046
Epoch 22/50
281/281 [==============================] - 44s 155ms/step - loss: 1.0668 - accuracy: 0.6246 - val_loss: 0.7648 - val_accuracy: 0.7440
Epoch 23/50
281/281 [==============================] - 43s 154ms/step - loss: 1.0651 - accuracy: 0.6174 - val_loss: 0.6847 - val_accuracy: 0.7702
Epoch 24/50
281/281 [==============================] - 43s 154ms/step - loss: 1.0406 - accuracy: 0.6260 - val_loss: 0.7319 - val_accuracy: 0.7440
Epoch 25/50
281/281 [==============================] - 43s 153ms/step - loss: 1.0429 - accuracy: 0.6339 - val_loss: 0.7684 - val_accuracy: 0.7117
Epoch 26/50
281/281 [==============================] - 43s 154ms/step - loss: 0.9944 - accuracy: 0.6392 - val_loss: 0.6802 - val_accuracy: 0.7510
Epoch 27/50
281/281 [==============================] - 43s 154ms/step - loss: 0.9763 - accuracy: 0.6437 - val_loss: 0.7534 - val_accuracy: 0.7248
Epoch 28/50
281/281 [==============================] - 43s 154ms/step - loss: 1.0191 - accuracy: 0.6293 - val_loss: 0.6837 - val_accuracy: 0.7671
Epoch 29/50
281/281 [==============================] - 43s 154ms/step - loss: 0.9727 - accuracy: 0.6614 - val_loss: 0.6194 - val_accuracy: 0.7954
Epoch 30/50
281/281 [==============================] - 43s 155ms/step - loss: 0.9770 - accuracy: 0.6561 - val_loss: 0.5944 - val_accuracy: 0.7893
Epoch 31/50
281/281 [==============================] - 43s 154ms/step - loss: 0.9488 - accuracy: 0.6653 - val_loss: 0.5919 - val_accuracy: 0.8004
Epoch 32/50
281/281 [==============================] - 43s 154ms/step - loss: 0.9309 - accuracy: 0.6719 - val_loss: 0.6000 - val_accuracy: 0.7863
Epoch 33/50
281/281 [==============================] - 43s 155ms/step - loss: 0.9452 - accuracy: 0.6639 - val_loss: 0.5757 - val_accuracy: 0.8115
Epoch 34/50
281/281 [==============================] - 43s 154ms/step - loss: 0.9118 - accuracy: 0.6746 - val_loss: 0.7505 - val_accuracy: 0.7036
Epoch 35/50
281/281 [==============================] - 43s 154ms/step - loss: 0.9049 - accuracy: 0.6848 - val_loss: 0.6069 - val_accuracy: 0.7802
Epoch 36/50
281/281 [==============================] - 43s 154ms/step - loss: 0.8735 - accuracy: 0.6912 - val_loss: 0.5430 - val_accuracy: 0.8196
Epoch 37/50
281/281 [==============================] - 43s 154ms/step - loss: 0.8792 - accuracy: 0.6901 - val_loss: 0.5772 - val_accuracy: 0.7964
Epoch 38/50
281/281 [==============================] - 43s 154ms/step - loss: 0.8981 - accuracy: 0.6877 - val_loss: 0.5409 - val_accuracy: 0.8014
Epoch 39/50
281/281 [==============================] - 43s 153ms/step - loss: 0.8530 - accuracy: 0.6992 - val_loss: 0.6134 - val_accuracy: 0.7812
Epoch 40/50
281/281 [==============================] - 43s 154ms/step - loss: 0.8615 - accuracy: 0.6927 - val_loss: 0.5690 - val_accuracy: 0.7994
Epoch 41/50
281/281 [==============================] - 43s 155ms/step - loss: 0.8593 - accuracy: 0.7002 - val_loss: 0.6011 - val_accuracy: 0.7782
Epoch 42/50
281/281 [==============================] - 43s 154ms/step - loss: 0.8398 - accuracy: 0.7064 - val_loss: 0.4951 - val_accuracy: 0.8105
Epoch 43/50
281/281 [==============================] - 43s 153ms/step - loss: 0.8406 - accuracy: 0.7099 - val_loss: 0.5005 - val_accuracy: 0.8266
Epoch 44/50
281/281 [==============================] - 43s 153ms/step - loss: 0.8268 - accuracy: 0.7109 - val_loss: 0.6435 - val_accuracy: 0.7651
Epoch 45/50
281/281 [==============================] - 43s 155ms/step - loss: 0.8306 - accuracy: 0.7068 - val_loss: 0.5801 - val_accuracy: 0.8054
Epoch 46/50
281/281 [==============================] - 43s 154ms/step - loss: 0.8172 - accuracy: 0.7139 - val_loss: 0.4901 - val_accuracy: 0.8417
Epoch 47/50
281/281 [==============================] - 43s 155ms/step - loss: 0.7891 - accuracy: 0.7229 - val_loss: 0.4869 - val_accuracy: 0.8236
Epoch 48/50
281/281 [==============================] - 43s 154ms/step - loss: 0.7749 - accuracy: 0.7327 - val_loss: 0.4772 - val_accuracy: 0.8286
Epoch 49/50
281/281 [==============================] - 43s 154ms/step - loss: 0.7961 - accuracy: 0.7225 - val_loss: 0.4985 - val_accuracy: 0.8286
Epoch 50/50
281/281 [==============================] - 43s 153ms/step - loss: 0.8169 - accuracy: 0.7172 - val_loss: 0.4563 - val_accuracy: 0.8377
In [270]:
plot_history(history)
yhat = np.argmax(resnet1.predict(x_test), axis=1)
acc = mt.accuracy_score(y_test,yhat)
cm = mt.confusion_matrix(y_test,yhat)
plot_confusion_matrix(cm,list(label.values()),title= f"CM Test Acc:{acc}")
The goal to change the drop out layer was to see the change of drop out in learning, we can see that if the drop is increased then there is more variation in the learning curve itself (more spikes) but it does not change the underfitting we are seeing. 


Changing Hyperparameter for Xception style network.

Here I am testing the change in depth multiplier in seperable convolution layer and Adam optmizer paramerters.

In [247]:
#  Xception 2

def build_net(depth_multiplier,learning_rate,beta_1,beta_2,epsilon,l2_lambda,dropout):
    
    input_holder = Input(shape=(img_wh, img_wh, 3))

    # start with a conv layer
    x = Conv2D(filters=32,
                   input_shape = (img_wh,img_wh,3), # lets keep the 3 channels
                   kernel_size=(3,3),
                   kernel_initializer='he_uniform', 
                   kernel_regularizer=l2(l2_lambda),
                   padding='same', 
                   activation='relu', 
                   data_format="channels_last")(input_holder)

    x = MaxPooling2D(pool_size=(2, 2), data_format="channels_last")(x)

    x = Conv2D(filters=32,
                   kernel_size=(3,3),
                   kernel_initializer='he_uniform', 
                   kernel_regularizer=l2(l2_lambda),
                   padding='same', 
                   activation='relu', 
                   data_format="channels_last")(x)

    x_split = MaxPooling2D(pool_size=(1, 1), data_format="channels_last")(x)

    x_split = SeparableConv2D(filters=32,
                   input_shape = (img_wh,img_wh,3),
                   kernel_size=(3,3),
                   kernel_initializer='he_uniform', 
                   kernel_regularizer=l2(l2_lambda),
                   padding='same', 
                   activation='relu', 
                   depth_multiplier = depth_multiplier, # controls output channels
                   data_format="channels_last")(x_split)


    x = Add()([x, x_split]) 
    
    # add 1x1 and activation layer
    
    x_split = MaxPooling2D(pool_size=(1, 1), data_format="channels_last")(x)

    x_split = Activation("relu")(x_split)
    

    x_split = SeparableConv2D(filters=32,
               input_shape = (img_wh,img_wh,1),
               kernel_size=(3,3),
               kernel_initializer='he_uniform', 
               kernel_regularizer=l2(l2_lambda),
               padding='same', 
               activation='relu', 
               depth_multiplier = depth_multiplier, # controls output channels
               data_format="channels_last")(x_split)
    

    x = Add()([x, x_split]) 
    
    x = Activation("relu")(x)

    x = MaxPooling2D(pool_size=(2, 2), data_format="channels_last")(x)

    
    x = Flatten()(x)
    x = Dropout(dropout)(x)
    x = Dense(64, activation="relu")(x)
    x = Dropout(dropout)(x)
    x = Dense(32, activation="relu")(x)
    x = Dropout(dropout)(x)
    x = Dense(NUM_CLASSES,activation="softmax")(x)

    xception2 = Model(inputs=input_holder,outputs=x)
    xception2.compile(loss='categorical_crossentropy', # 'categorical_crossentropy' 'mean_squared_error'
                    optimizer=tf.keras.optimizers.Adam(
                                                learning_rate=learning_rate,
                                                beta_1=beta_1,
                                                beta_2=beta_2,
                                                epsilon=epsilon,
#                                                 amsgrad=False,
                                                name="Adam",), # 'adadelta' 'rmsprop'
                    metrics=['accuracy'])
    return xception2
#     xception.summary()

    
In [248]:
xception2 = build_net(depth_multiplier=3,learning_rate=0.001,beta_1=0.9,beta_2=0.999,epsilon=0.00001,l2_lambda=0.000001,dropout=0.5)
xception2.summary()
Model: "model_10"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_12 (InputLayer)           [(None, 128, 128, 3) 0                                            
__________________________________________________________________________________________________
conv2d_81 (Conv2D)              (None, 128, 128, 32) 896         input_12[0][0]                   
__________________________________________________________________________________________________
max_pooling2d_57 (MaxPooling2D) (None, 64, 64, 32)   0           conv2d_81[0][0]                  
__________________________________________________________________________________________________
conv2d_82 (Conv2D)              (None, 64, 64, 32)   9248        max_pooling2d_57[0][0]           
__________________________________________________________________________________________________
max_pooling2d_58 (MaxPooling2D) (None, 64, 64, 32)   0           conv2d_82[0][0]                  
__________________________________________________________________________________________________
separable_conv2d_13 (SeparableC (None, 64, 64, 32)   3968        max_pooling2d_58[0][0]           
__________________________________________________________________________________________________
add_14 (Add)                    (None, 64, 64, 32)   0           conv2d_82[0][0]                  
                                                                 separable_conv2d_13[0][0]        
__________________________________________________________________________________________________
max_pooling2d_59 (MaxPooling2D) (None, 64, 64, 32)   0           add_14[0][0]                     
__________________________________________________________________________________________________
activation_14 (Activation)      (None, 64, 64, 32)   0           max_pooling2d_59[0][0]           
__________________________________________________________________________________________________
separable_conv2d_14 (SeparableC (None, 64, 64, 32)   3968        activation_14[0][0]              
__________________________________________________________________________________________________
add_15 (Add)                    (None, 64, 64, 32)   0           add_14[0][0]                     
                                                                 separable_conv2d_14[0][0]        
__________________________________________________________________________________________________
activation_15 (Activation)      (None, 64, 64, 32)   0           add_15[0][0]                     
__________________________________________________________________________________________________
max_pooling2d_60 (MaxPooling2D) (None, 32, 32, 32)   0           activation_15[0][0]              
__________________________________________________________________________________________________
flatten_22 (Flatten)            (None, 32768)        0           max_pooling2d_60[0][0]           
__________________________________________________________________________________________________
dropout_40 (Dropout)            (None, 32768)        0           flatten_22[0][0]                 
__________________________________________________________________________________________________
dense_52 (Dense)                (None, 64)           2097216     dropout_40[0][0]                 
__________________________________________________________________________________________________
dropout_41 (Dropout)            (None, 64)           0           dense_52[0][0]                   
__________________________________________________________________________________________________
dense_53 (Dense)                (None, 32)           2080        dropout_41[0][0]                 
__________________________________________________________________________________________________
dropout_42 (Dropout)            (None, 32)           0           dense_53[0][0]                   
__________________________________________________________________________________________________
dense_54 (Dense)                (None, 10)           330         dropout_42[0][0]                 
==================================================================================================
Total params: 2,117,706
Trainable params: 2,117,706
Non-trainable params: 0
__________________________________________________________________________________________________
In [249]:
BS  = 32
epochs = 10
history = xception2.fit_generator(datagen_train.flow(X_train, Y_train, batch_size=BS), 
                   steps_per_epoch=len(X_train)//BS, # how many generators to go through per epoch
                   epochs=epochs, verbose=1,
                   validation_data=datagen_val.flow(X_val,Y_val, batch_size=BS),
                   validation_steps=len(X_val)//BS,
                   callbacks=[EarlyStopping(monitor='val_loss', patience=4)],
                  )
Epoch 1/10
281/281 [==============================] - 45s 156ms/step - loss: 2.3818 - accuracy: 0.0975 - val_loss: 2.2861 - val_accuracy: 0.1613
Epoch 2/10
281/281 [==============================] - 43s 154ms/step - loss: 2.3028 - accuracy: 0.0961 - val_loss: 2.3026 - val_accuracy: 0.1018
Epoch 3/10
281/281 [==============================] - 43s 153ms/step - loss: 2.3012 - accuracy: 0.0987 - val_loss: 2.2954 - val_accuracy: 0.1119
Epoch 4/10
281/281 [==============================] - 43s 153ms/step - loss: 2.2867 - accuracy: 0.1210 - val_loss: 2.2513 - val_accuracy: 0.1875
Epoch 5/10
281/281 [==============================] - 43s 154ms/step - loss: 2.2419 - accuracy: 0.1498 - val_loss: 2.1836 - val_accuracy: 0.2339
Epoch 6/10
281/281 [==============================] - 44s 155ms/step - loss: 2.2161 - accuracy: 0.1723 - val_loss: 2.1646 - val_accuracy: 0.2117
Epoch 7/10
281/281 [==============================] - 43s 154ms/step - loss: 2.1686 - accuracy: 0.1966 - val_loss: 2.0430 - val_accuracy: 0.2651
Epoch 8/10
281/281 [==============================] - 43s 153ms/step - loss: 2.1054 - accuracy: 0.2259 - val_loss: 1.9670 - val_accuracy: 0.3296
Epoch 9/10
281/281 [==============================] - 43s 153ms/step - loss: 2.0572 - accuracy: 0.2504 - val_loss: 1.9133 - val_accuracy: 0.3306
Epoch 10/10
281/281 [==============================] - 43s 155ms/step - loss: 2.0413 - accuracy: 0.2513 - val_loss: 1.9696 - val_accuracy: 0.3206
In [250]:
plot_history(history)
yhat = np.argmax(xception2.predict(x_test), axis=1)
acc = mt.accuracy_score(y_test,yhat)
cm = mt.confusion_matrix(y_test,yhat)
plot_confusion_matrix(cm,list(label.values()),title= f"CM Test Acc:{acc}")
Just testing random combination of hyperparameter, atleast the loss in decresing.

Larger Image Size

The motivation to change the image size is becasue AlexNet[2] is using images for size 224x224x3 so I decided that since I am testing AlexNet style newtwork I will use the same size images.

In [54]:
TEST_DIR="/users/sbataju/CS5324/tomato/val/"


NUM_CLASSES = 10
index=0
data=[]
for FOLDER in os.listdir(TEST_DIR):
    print(FOLDER,inv_label[FOLDER])
    for image_dir in os.listdir(TEST_DIR+FOLDER):
        data.append({"x":img_to_np(TEST_DIR+FOLDER+"/"+image_dir,flatten=False,newsize=(224,224)),"y":label[inv_label[FOLDER]]})
#         assert False
    index=index+1
    
x,y=[],[]
for obj in data:
    x.append(obj["x"])
    y.append(obj["y"])
x_test1 = np.array(x)
y_test1 = np.array([inv_label[i] for i in y],dtype=np.float16)  
Tomato___Tomato_mosaic_virus 0
Tomato___Spider_mites Two-spotted_spider_mite 1
Tomato___Leaf_Mold 2
Tomato___Early_blight 3
Tomato___Target_Spot 4
Tomato___Late_blight 5
Tomato___healthy 6
Tomato___Bacterial_spot 7
Tomato___Tomato_Yellow_Leaf_Curl_Virus 8
Tomato___Septoria_leaf_spot 9
In [55]:
TRAIN_DIR = "/users/sbataju/CS5324/tomato/train/"
index=0
data=[]
for FOLDER in os.listdir(TRAIN_DIR):
    print(FOLDER,'  ',inv_label[FOLDER])
    for image_dir in os.listdir(TRAIN_DIR+FOLDER):
        data.append({"x":img_to_np(TRAIN_DIR+FOLDER+"/"+image_dir,flatten=False,newsize=(224,224)),"y":label[inv_label[FOLDER]]})
    index=index+1
x,y=[],[]
for obj in data:
    x.append(obj["x"])
    y.append(obj["y"])
x_train1 = np.array(x)
y_train1 = np.array([inv_label[i] for i in y],dtype=np.float16)  

y_train_ohe1 = keras.utils.to_categorical(y_train, NUM_CLASSES)
y_test_ohe1 = keras.utils.to_categorical(y_test, NUM_CLASSES)
Tomato___Tomato_mosaic_virus    0
Tomato___healthy    6
Tomato___Tomato_Yellow_Leaf_Curl_Virus    8
Tomato___Septoria_leaf_spot    9
Tomato___Late_blight    5
Tomato___Early_blight    3
Tomato___Leaf_Mold    2
Tomato___Spider_mites Two-spotted_spider_mite    1
Tomato___Target_Spot    4
Tomato___Bacterial_spot    7
In [56]:
from sklearn.model_selection import train_test_split
X_train1, X_val1, Y_train1, Y_val1 = train_test_split(
    x_train1, y_train_ohe1, test_size=0.1, random_state=42,stratify=y_train,)
In [57]:
X_train1,X_val1 =X_train1/255.0 , X_val1/255.0 
x_test1 = x_test1/255.0
In [58]:
datagen_train1 = ImageDataGenerator(
    featurewise_center=False,
    samplewise_center=False,
    featurewise_std_normalization=False,
    samplewise_std_normalization=False,
    zca_whitening=False,
    rotation_range=180, # used, Int. Degree range for random rotations.
    width_shift_range=0.2, # used, Float (fraction of total width). Range for random horizontal shifts.
    height_shift_range=0.2, # used,  Float (fraction of total height). Range for random vertical shifts.
    shear_range=0.1, # Float. Shear Intensity (Shear angle in counter-clockwise direction as radians)
    zoom_range=0.0,
    channel_shift_range=0.,
    fill_mode='nearest',
    cval=0.,
#     brightness_range=[0.2,1],
    horizontal_flip=True,
    vertical_flip=True,
    rescale=None)


datagen_val1 = ImageDataGenerator(
    featurewise_center=False,
    samplewise_center=False,
    featurewise_std_normalization=False,
    samplewise_std_normalization=False,
    zca_whitening=False,
    rotation_range=180, # used, Int. Degree range for random rotations.
    width_shift_range=0.2, # used, Float (fraction of total width). Range for random horizontal shifts.
    height_shift_range=0.2, # used,  Float (fraction of total height). Range for random vertical shifts.
    shear_range=0.1, # Float. Shear Intensity (Shear angle in counter-clockwise direction as radians)
    zoom_range=0.0,
    channel_shift_range=0.,
    fill_mode='nearest',
    cval=0.,
#     brightness_range=[0.2,1],
    horizontal_flip=True,
    vertical_flip=True,
    rescale=None)

datagen_train1.fit(X_train1)
datagen_val1.fit(X_val1)
In [59]:
tmps1 = datagen_train1.flow(X_train1, Y_train1, batch_size=3)



for i,tmp_ in enumerate(tmps1):
#     imshow(tmp[i].squeeze(),cmap='bone')
    
    for j,tmp in enumerate(tmp_):
        if j == 0:
            for k,t in enumerate(tmp):
                plt.subplot(1,3,k+1)
                plt.imshow(t)
                plt.grid()
        else:
            plt.title(label[np.argmax(tmp)])
    if i ==10:  break
    plt.show()    
In [61]:
X_train1[0].shape
Out[61]:
(224, 224, 3)
I applied all the same perprocessing steps as before and we can see the images are of 224x224x3.


Complex Networks

Now let's first test adding batch normalization layer after every convolution layers to the ResNet we had before.

In [104]:
fig = plt.figure(figsize=(60,6))
img = mpimg.imread('resnet_batch1.png')
plt.imshow(img)

plt.show()
In [128]:
%%time
#https://www.kdnuggets.com/2018/09/dropout-convolutional-networks.html

from tensorflow.keras.layers import Add, Input
from tensorflow.keras.layers import average, concatenate
from tensorflow.keras.models import Model
from tensorflow.keras.regularizers import l2
l2_lambda = 0.0001
img_wh = 224
input_holder = Input(shape=(img_wh, img_wh, 3))

# start with a conv layer
x = Conv2D(filters=32,
               input_shape = (img_wh,img_wh,1),
               kernel_size=(3,3),
               kernel_initializer='he_uniform', 
               kernel_regularizer=l2(l2_lambda),
               padding='same', 
               activation='relu', 
               data_format="channels_last")(input_holder)
x = layers.BatchNormalization(axis=-1)(x)

x = MaxPooling2D(pool_size=(2, 2), data_format="channels_last")(x)

x = Conv2D(filters=32,
               kernel_size=(3,3),
               kernel_initializer='he_uniform', 
               kernel_regularizer=l2(l2_lambda),
               padding='same', 
               activation='relu', 
               data_format="channels_last")(x)
x = layers.BatchNormalization(axis=-1)(x)

x_split = MaxPooling2D(pool_size=(2, 2), data_format="channels_last")(x)

x = Conv2D(filters=64,
               kernel_size=(1,1),
               kernel_initializer='he_uniform', 
               kernel_regularizer=l2(l2_lambda),
               padding='same', 
               activation='LeakyReLU', 
               data_format="channels_last")(x_split)
x = layers.BatchNormalization(axis=-1)(x)

x = Conv2D(filters=64,
               kernel_size=(3,3),
               kernel_initializer='he_uniform', 
               kernel_regularizer=l2(l2_lambda),
               padding='same', 
               activation='LeakyReLU', 
               data_format="channels_last")(x)
x = layers.BatchNormalization(axis=-1)(x)

x = Conv2D(filters=32,
               kernel_size=(1,1),
               kernel_initializer='he_uniform', 
               kernel_regularizer=l2(l2_lambda),
               padding='same', 
               activation='LeakyReLU', 
               data_format="channels_last")(x)
x = layers.BatchNormalization(axis=-1)(x)

# now add back in the split layer, x_split (residual added in)
x = Add()([x, x_split])

x = Activation("LeakyReLU")(x)

x = MaxPooling2D(pool_size=(2, 2), data_format="channels_last")(x)

x = Flatten()(x)
x = Dropout(0.25)(x)
x = Dense(256)(x)
x = Activation("relu")(x)
x = Dropout(0.5)(x)
x = Dense(NUM_CLASSES)(x)
x = Activation('softmax')(x)

resnet_batch1 = Model(inputs=input_holder,outputs=x)

resnet_batch1.summary()
Model: "model_6"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_9 (InputLayer)            [(None, 224, 224, 3) 0                                            
__________________________________________________________________________________________________
conv2d_30 (Conv2D)              (None, 224, 224, 32) 896         input_9[0][0]                    
__________________________________________________________________________________________________
batch_normalization_35 (BatchNo (None, 224, 224, 32) 128         conv2d_30[0][0]                  
__________________________________________________________________________________________________
max_pooling2d_18 (MaxPooling2D) (None, 112, 112, 32) 0           batch_normalization_35[0][0]     
__________________________________________________________________________________________________
conv2d_31 (Conv2D)              (None, 112, 112, 32) 9248        max_pooling2d_18[0][0]           
__________________________________________________________________________________________________
batch_normalization_36 (BatchNo (None, 112, 112, 32) 128         conv2d_31[0][0]                  
__________________________________________________________________________________________________
max_pooling2d_19 (MaxPooling2D) (None, 56, 56, 32)   0           batch_normalization_36[0][0]     
__________________________________________________________________________________________________
conv2d_32 (Conv2D)              (None, 56, 56, 64)   2112        max_pooling2d_19[0][0]           
__________________________________________________________________________________________________
batch_normalization_37 (BatchNo (None, 56, 56, 64)   256         conv2d_32[0][0]                  
__________________________________________________________________________________________________
conv2d_33 (Conv2D)              (None, 56, 56, 64)   36928       batch_normalization_37[0][0]     
__________________________________________________________________________________________________
batch_normalization_38 (BatchNo (None, 56, 56, 64)   256         conv2d_33[0][0]                  
__________________________________________________________________________________________________
conv2d_34 (Conv2D)              (None, 56, 56, 32)   2080        batch_normalization_38[0][0]     
__________________________________________________________________________________________________
batch_normalization_39 (BatchNo (None, 56, 56, 32)   128         conv2d_34[0][0]                  
__________________________________________________________________________________________________
add_6 (Add)                     (None, 56, 56, 32)   0           batch_normalization_39[0][0]     
                                                                 max_pooling2d_19[0][0]           
__________________________________________________________________________________________________
activation_18 (Activation)      (None, 56, 56, 32)   0           add_6[0][0]                      
__________________________________________________________________________________________________
max_pooling2d_20 (MaxPooling2D) (None, 28, 28, 32)   0           activation_18[0][0]              
__________________________________________________________________________________________________
flatten_6 (Flatten)             (None, 25088)        0           max_pooling2d_20[0][0]           
__________________________________________________________________________________________________
dropout_12 (Dropout)            (None, 25088)        0           flatten_6[0][0]                  
__________________________________________________________________________________________________
dense_12 (Dense)                (None, 256)          6422784     dropout_12[0][0]                 
__________________________________________________________________________________________________
activation_19 (Activation)      (None, 256)          0           dense_12[0][0]                   
__________________________________________________________________________________________________
dropout_13 (Dropout)            (None, 256)          0           activation_19[0][0]              
__________________________________________________________________________________________________
dense_13 (Dense)                (None, 10)           2570        dropout_13[0][0]                 
__________________________________________________________________________________________________
activation_20 (Activation)      (None, 10)           0           dense_13[0][0]                   
==================================================================================================
Total params: 6,477,514
Trainable params: 6,477,066
Non-trainable params: 448
__________________________________________________________________________________________________
CPU times: user 212 ms, sys: 5.23 ms, total: 217 ms
Wall time: 214 ms
In [130]:
resnet_batch1.compile(loss='categorical_crossentropy', # 'categorical_crossentropy' 'mean_squared_error'
                optimizer='adam', # 'adadelta' 'rmsprop'
                metrics=['accuracy'])
In [131]:
BS  = 32
epochs = 50
history = resnet_batch1.fit_generator(datagen_train1.flow(X_train1, Y_train1, batch_size=BS), 
                   steps_per_epoch=len(X_train1)//BS, # how many generators to go through per epoch
                   epochs=epochs, verbose=1,
                   validation_data=datagen_val1.flow(X_val1,Y_val1, batch_size=BS),
                   validation_steps=len(X_val1)//BS,
                   callbacks=[EarlyStopping(monitor='val_loss', patience=5)],
                  )
Epoch 1/50
281/281 [==============================] - 109s 381ms/step - loss: 3.9536 - accuracy: 0.2324 - val_loss: 2.2835 - val_accuracy: 0.2077
Epoch 2/50
281/281 [==============================] - 108s 382ms/step - loss: 1.8229 - accuracy: 0.3630 - val_loss: 1.6386 - val_accuracy: 0.4597
Epoch 3/50
281/281 [==============================] - 105s 371ms/step - loss: 1.6759 - accuracy: 0.4069 - val_loss: 1.4450 - val_accuracy: 0.4980
Epoch 4/50
281/281 [==============================] - 107s 381ms/step - loss: 1.5289 - accuracy: 0.4548 - val_loss: 1.1503 - val_accuracy: 0.6240
Epoch 5/50
281/281 [==============================] - 89s 317ms/step - loss: 1.4454 - accuracy: 0.4797 - val_loss: 1.5709 - val_accuracy: 0.5363
Epoch 6/50
281/281 [==============================] - 109s 386ms/step - loss: 1.3237 - accuracy: 0.5231 - val_loss: 1.1613 - val_accuracy: 0.6371
Epoch 7/50
281/281 [==============================] - 108s 384ms/step - loss: 1.2410 - accuracy: 0.5569 - val_loss: 0.9545 - val_accuracy: 0.6562
Epoch 8/50
281/281 [==============================] - 107s 380ms/step - loss: 1.2025 - accuracy: 0.5734 - val_loss: 1.0987 - val_accuracy: 0.6583
Epoch 9/50
281/281 [==============================] - 108s 382ms/step - loss: 1.1691 - accuracy: 0.5956 - val_loss: 1.5962 - val_accuracy: 0.5847
Epoch 10/50
281/281 [==============================] - 108s 384ms/step - loss: 1.0691 - accuracy: 0.6289 - val_loss: 1.1239 - val_accuracy: 0.6835
Epoch 11/50
281/281 [==============================] - 108s 382ms/step - loss: 1.0057 - accuracy: 0.6525 - val_loss: 0.7685 - val_accuracy: 0.7530
Epoch 12/50
281/281 [==============================] - 108s 383ms/step - loss: 0.9760 - accuracy: 0.6684 - val_loss: 0.6479 - val_accuracy: 0.7833
Epoch 13/50
281/281 [==============================] - 109s 386ms/step - loss: 0.9250 - accuracy: 0.6979 - val_loss: 0.8819 - val_accuracy: 0.6915
Epoch 14/50
281/281 [==============================] - 108s 386ms/step - loss: 0.8869 - accuracy: 0.7051 - val_loss: 0.5067 - val_accuracy: 0.8498
Epoch 15/50
281/281 [==============================] - 108s 385ms/step - loss: 0.8315 - accuracy: 0.7259 - val_loss: 1.2609 - val_accuracy: 0.7167
Epoch 16/50
281/281 [==============================] - 108s 385ms/step - loss: 0.8192 - accuracy: 0.7331 - val_loss: 0.5824 - val_accuracy: 0.8276
Epoch 17/50
281/281 [==============================] - 110s 391ms/step - loss: 0.7956 - accuracy: 0.7377 - val_loss: 0.6645 - val_accuracy: 0.7833
Epoch 18/50
281/281 [==============================] - 108s 385ms/step - loss: 0.7816 - accuracy: 0.7503 - val_loss: 0.7791 - val_accuracy: 0.7470
Epoch 19/50
281/281 [==============================] - 108s 385ms/step - loss: 0.7250 - accuracy: 0.7690 - val_loss: 0.4986 - val_accuracy: 0.8508
Epoch 20/50
281/281 [==============================] - 108s 384ms/step - loss: 0.7325 - accuracy: 0.7653 - val_loss: 1.8530 - val_accuracy: 0.5726
Epoch 21/50
281/281 [==============================] - 108s 383ms/step - loss: 0.7263 - accuracy: 0.7686 - val_loss: 0.6169 - val_accuracy: 0.8236
Epoch 22/50
281/281 [==============================] - 108s 384ms/step - loss: 0.6840 - accuracy: 0.7800 - val_loss: 0.4027 - val_accuracy: 0.8790
Epoch 23/50
281/281 [==============================] - 108s 386ms/step - loss: 0.6961 - accuracy: 0.7874 - val_loss: 0.6396 - val_accuracy: 0.8024
Epoch 24/50
281/281 [==============================] - 106s 376ms/step - loss: 0.6647 - accuracy: 0.7891 - val_loss: 0.4454 - val_accuracy: 0.8710
Epoch 25/50
281/281 [==============================] - 109s 386ms/step - loss: 0.6312 - accuracy: 0.8029 - val_loss: 1.3666 - val_accuracy: 0.7329
Epoch 26/50
281/281 [==============================] - 107s 382ms/step - loss: 0.6190 - accuracy: 0.8139 - val_loss: 0.4671 - val_accuracy: 0.8700
Epoch 27/50
281/281 [==============================] - 107s 382ms/step - loss: 0.6114 - accuracy: 0.8116 - val_loss: 0.4120 - val_accuracy: 0.8800
In [134]:
plot_history(history)
yhat = np.argmax(resnet_batch1.predict(x_test1), axis=1)
acc = mt.accuracy_score(y_test1,yhat)
cm = mt.confusion_matrix(y_test1,yhat)
plot_confusion_matrix(cm,list(label.values()),title= f"CM Test Acc:{acc}")
Looking at the epoch vs accuracy graphs we can see the underfitting is significantly reduced, there are spikes in validation set but unlike before the validation learning curve is not always greater than the training learning curve. Furthermore, the training curve is continously decreasing as well.
Uptill now I think this network style is most promising given the learning curve so, I am choosing this network to optimize the hyperparameter.

Hyperparameter Optmization

Here is the optmization preformed using Optuna the code used to get these plot is in the bottom of the report. 
There is no significant imporvement in accuracy compared to our defult model.
In [120]:
plt.figure(figsize=(18,18))
plt.subplot(1,2,1)
img = mpimg.imread('resnet_batch.png')
plt.imshow(img)
# plt.show()
plt.subplot(1,2,2)
img = mpimg.imread('plot_parallel_coordinate_resnet_batch.png')
plt.imshow(img)
plt.show()
plt.figure(figsize=(18,18))

img = mpimg.imread('plot_slice_resnet_batch.png')
plt.imshow(img)
plt.show()
plt.figure(figsize=(18,18))

img = mpimg.imread('plot_contour_resnet_batch.png')
plt.imshow(img)
plt.show()
plt.figure(figsize=(18,18))

img = mpimg.imread('plot_param_importances_resnet_batch.png')
plt.imshow(img)
plt.show()

Add and Subtract ResNet

I wanted to test what would happend if we not only add the convolution layer to the starting layer but also substract from the starting layer and concatenate all the layer. So that not only features are being encouraged but also non features are also being encouraged. 



In [105]:
fig = plt.figure(figsize=(60,6))
img = mpimg.imread('resnet_addsub.png')
plt.imshow(img)
plt.show()
In [33]:
%%time
img_wh = 224
from tensorflow.keras.layers import Add, Input, Subtract
from tensorflow.keras.layers import average, concatenate
from tensorflow.keras.models import Model

input_holder = Input(shape=(img_wh, img_wh, 3))
l2_lambda = 0.0001
img_wh = 224
# start with a conv layer
x = Conv2D(filters=32,
               input_shape = (img_wh,img_wh,1),
               kernel_size=(3,3),
               kernel_initializer='he_uniform', 
               kernel_regularizer=l2(l2_lambda),
               padding='same', 
               activation='relu', 
               data_format="channels_last")(input_holder)

x = MaxPooling2D(pool_size=(2, 2), data_format="channels_last")(x)

x = Conv2D(filters=32,
               kernel_size=(3,3),
               kernel_initializer='he_uniform', 
               kernel_regularizer=l2(l2_lambda),
               padding='same', 
               activation='relu', 
               data_format="channels_last")(x)

x_split = MaxPooling2D(pool_size=(2, 2), data_format="channels_last")(x)

# For add block
x = Conv2D(filters=64,
               kernel_size=(1,1),
               kernel_initializer='he_uniform', 
               kernel_regularizer=l2(l2_lambda),
               padding='same', 
               activation='relu', 
               data_format="channels_last")(x_split)

x = Conv2D(filters=64,
               kernel_size=(3,3),
               kernel_initializer='he_uniform', 
               kernel_regularizer=l2(l2_lambda),
               padding='same', 
               activation='relu', 
               data_format="channels_last")(x)

x = Conv2D(filters=32,
               kernel_size=(1,1),
               kernel_initializer='he_uniform', 
               kernel_regularizer=l2(l2_lambda),
               padding='same', 
               activation='relu', 
               data_format="channels_last")(x)

# For subtract block 
x1 = Conv2D(filters=64,
               kernel_size=(1,1),
               kernel_initializer='he_uniform', 
               kernel_regularizer=l2(l2_lambda),
               padding='same', 
               activation='relu', 
               data_format="channels_last")(x_split)

x1 = Conv2D(filters=64,
               kernel_size=(3,3),
               kernel_initializer='he_uniform', 
               kernel_regularizer=l2(l2_lambda),
               padding='same', 
               activation='relu', 
               data_format="channels_last")(x1)

x1 = Conv2D(filters=32,
               kernel_size=(1,1),
               kernel_initializer='he_uniform', 
               kernel_regularizer=l2(l2_lambda),
               padding='same', 
               activation='relu', 
               data_format="channels_last")(x1)


 
# now add back in the split layer, x_split (residual added in)
x = Add()([x, x_split])
x1 = Subtract()([x_split,x1])
x = concatenate([x,x1])
x = layers.BatchNormalization(axis=-1)(x)

x = Activation("relu")(x)

x = MaxPooling2D(pool_size=(2, 2), data_format="channels_last")(x)

x = Flatten()(x)
x = Dropout(0.25)(x)
x = Dense(256)(x)
x = Activation("relu")(x)
x = Dropout(0.5)(x)
x = Dense(NUM_CLASSES)(x)
x = Activation('softmax')(x)

resnet_addsub = Model(inputs=input_holder,outputs=x)

resnet_addsub.summary()
Model: "model"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_3 (InputLayer)            [(None, 224, 224, 3) 0                                            
__________________________________________________________________________________________________
conv2d_8 (Conv2D)               (None, 224, 224, 32) 896         input_3[0][0]                    
__________________________________________________________________________________________________
max_pooling2d_2 (MaxPooling2D)  (None, 112, 112, 32) 0           conv2d_8[0][0]                   
__________________________________________________________________________________________________
conv2d_9 (Conv2D)               (None, 112, 112, 32) 9248        max_pooling2d_2[0][0]            
__________________________________________________________________________________________________
max_pooling2d_3 (MaxPooling2D)  (None, 56, 56, 32)   0           conv2d_9[0][0]                   
__________________________________________________________________________________________________
conv2d_10 (Conv2D)              (None, 56, 56, 64)   2112        max_pooling2d_3[0][0]            
__________________________________________________________________________________________________
conv2d_13 (Conv2D)              (None, 56, 56, 64)   2112        max_pooling2d_3[0][0]            
__________________________________________________________________________________________________
conv2d_11 (Conv2D)              (None, 56, 56, 64)   36928       conv2d_10[0][0]                  
__________________________________________________________________________________________________
conv2d_14 (Conv2D)              (None, 56, 56, 64)   36928       conv2d_13[0][0]                  
__________________________________________________________________________________________________
conv2d_12 (Conv2D)              (None, 56, 56, 32)   2080        conv2d_11[0][0]                  
__________________________________________________________________________________________________
conv2d_15 (Conv2D)              (None, 56, 56, 32)   2080        conv2d_14[0][0]                  
__________________________________________________________________________________________________
add_1 (Add)                     (None, 56, 56, 32)   0           conv2d_12[0][0]                  
                                                                 max_pooling2d_3[0][0]            
__________________________________________________________________________________________________
subtract (Subtract)             (None, 56, 56, 32)   0           max_pooling2d_3[0][0]            
                                                                 conv2d_15[0][0]                  
__________________________________________________________________________________________________
concatenate (Concatenate)       (None, 56, 56, 64)   0           add_1[0][0]                      
                                                                 subtract[0][0]                   
__________________________________________________________________________________________________
batch_normalization (BatchNorma (None, 56, 56, 64)   256         concatenate[0][0]                
__________________________________________________________________________________________________
activation (Activation)         (None, 56, 56, 64)   0           batch_normalization[0][0]        
__________________________________________________________________________________________________
max_pooling2d_4 (MaxPooling2D)  (None, 28, 28, 64)   0           activation[0][0]                 
__________________________________________________________________________________________________
flatten (Flatten)               (None, 50176)        0           max_pooling2d_4[0][0]            
__________________________________________________________________________________________________
dropout (Dropout)               (None, 50176)        0           flatten[0][0]                    
__________________________________________________________________________________________________
dense (Dense)                   (None, 256)          12845312    dropout[0][0]                    
__________________________________________________________________________________________________
activation_1 (Activation)       (None, 256)          0           dense[0][0]                      
__________________________________________________________________________________________________
dropout_1 (Dropout)             (None, 256)          0           activation_1[0][0]               
__________________________________________________________________________________________________
dense_1 (Dense)                 (None, 10)           2570        dropout_1[0][0]                  
__________________________________________________________________________________________________
activation_2 (Activation)       (None, 10)           0           dense_1[0][0]                    
==================================================================================================
Total params: 12,940,522
Trainable params: 12,940,394
Non-trainable params: 128
__________________________________________________________________________________________________
CPU times: user 137 ms, sys: 1.38 ms, total: 139 ms
Wall time: 137 ms
In [35]:
resnet_addsub.compile(loss='categorical_crossentropy', # 'categorical_crossentropy' 'mean_squared_error'
                optimizer='adam', # 'adadelta' 'rmsprop'
                metrics=['accuracy'])
In [36]:
BS  = 64
epochs = 50
history = resnet_addsub.fit_generator(datagen_train1.flow(X_train1, Y_train1, batch_size=BS), 
                   steps_per_epoch=len(X_train1)//BS, # how many generators to go through per epoch
                   epochs=epochs, verbose=1,
                   validation_data=datagen_val1.flow(X_val1,Y_val1, batch_size=BS),
                   validation_steps=len(X_val1)//BS,
                   callbacks=[EarlyStopping(monitor='val_loss', patience=5)],
                  )
Epoch 1/50
140/140 [==============================] - 106s 742ms/step - loss: 3.8511 - accuracy: 0.1387 - val_loss: 2.3104 - val_accuracy: 0.1583
Epoch 2/50
140/140 [==============================] - 105s 748ms/step - loss: 2.2238 - accuracy: 0.1749 - val_loss: 2.1318 - val_accuracy: 0.2677
Epoch 3/50
140/140 [==============================] - 104s 744ms/step - loss: 2.1308 - accuracy: 0.2106 - val_loss: 2.0889 - val_accuracy: 0.2333
Epoch 4/50
140/140 [==============================] - 103s 735ms/step - loss: 2.0909 - accuracy: 0.2181 - val_loss: 2.1880 - val_accuracy: 0.1813
Epoch 5/50
140/140 [==============================] - 104s 740ms/step - loss: 2.0230 - accuracy: 0.2407 - val_loss: 1.8671 - val_accuracy: 0.3344
Epoch 6/50
140/140 [==============================] - 104s 741ms/step - loss: 1.9756 - accuracy: 0.2659 - val_loss: 1.9102 - val_accuracy: 0.3365
Epoch 7/50
140/140 [==============================] - 104s 744ms/step - loss: 1.9510 - accuracy: 0.2770 - val_loss: 2.0810 - val_accuracy: 0.2198
Epoch 8/50
140/140 [==============================] - 104s 742ms/step - loss: 1.9221 - accuracy: 0.2863 - val_loss: 2.3923 - val_accuracy: 0.3104
Epoch 9/50
140/140 [==============================] - 103s 734ms/step - loss: 1.9016 - accuracy: 0.2911 - val_loss: 1.7811 - val_accuracy: 0.3750
Epoch 10/50
140/140 [==============================] - 104s 740ms/step - loss: 1.8803 - accuracy: 0.2949 - val_loss: 2.8437 - val_accuracy: 0.3073
Epoch 11/50
140/140 [==============================] - 103s 735ms/step - loss: 1.8511 - accuracy: 0.3020 - val_loss: 1.8086 - val_accuracy: 0.3281
Epoch 12/50
140/140 [==============================] - 102s 732ms/step - loss: 1.8429 - accuracy: 0.3039 - val_loss: 1.6683 - val_accuracy: 0.4323
Epoch 13/50
140/140 [==============================] - 103s 736ms/step - loss: 1.8455 - accuracy: 0.3066 - val_loss: 1.7279 - val_accuracy: 0.3812
Epoch 14/50
140/140 [==============================] - 103s 736ms/step - loss: 1.8151 - accuracy: 0.3180 - val_loss: 3.0038 - val_accuracy: 0.2802
Epoch 15/50
140/140 [==============================] - 103s 735ms/step - loss: 1.8055 - accuracy: 0.3278 - val_loss: 2.2396 - val_accuracy: 0.3031
Epoch 16/50
140/140 [==============================] - 103s 737ms/step - loss: 1.8019 - accuracy: 0.3220 - val_loss: 1.6670 - val_accuracy: 0.3844
Epoch 17/50
140/140 [==============================] - 103s 736ms/step - loss: 1.7850 - accuracy: 0.3305 - val_loss: 1.6330 - val_accuracy: 0.4354
Epoch 18/50
140/140 [==============================] - 103s 734ms/step - loss: 1.7683 - accuracy: 0.3386 - val_loss: 1.9884 - val_accuracy: 0.3990
Epoch 19/50
140/140 [==============================] - 103s 734ms/step - loss: 1.7537 - accuracy: 0.3419 - val_loss: 1.5863 - val_accuracy: 0.4396
Epoch 20/50
140/140 [==============================] - 102s 730ms/step - loss: 1.7232 - accuracy: 0.3542 - val_loss: 1.4565 - val_accuracy: 0.5240
Epoch 21/50
140/140 [==============================] - 102s 729ms/step - loss: 1.6914 - accuracy: 0.3744 - val_loss: 1.8006 - val_accuracy: 0.3875
Epoch 22/50
140/140 [==============================] - 103s 736ms/step - loss: 1.6592 - accuracy: 0.3804 - val_loss: 1.8499 - val_accuracy: 0.4250
Epoch 23/50
140/140 [==============================] - 103s 735ms/step - loss: 1.6456 - accuracy: 0.3910 - val_loss: 1.4600 - val_accuracy: 0.5562
Epoch 24/50
140/140 [==============================] - 102s 731ms/step - loss: 1.6262 - accuracy: 0.3967 - val_loss: 1.7390 - val_accuracy: 0.3375
Epoch 25/50
140/140 [==============================] - 102s 731ms/step - loss: 1.6200 - accuracy: 0.3974 - val_loss: 1.7031 - val_accuracy: 0.3490
In [37]:
plot_history(history)
yhat = np.argmax(resnet_addsub.predict(x_test1), axis=1)
acc = mt.accuracy_score(y_test1,yhat)
cm = mt.confusion_matrix(y_test1,yhat)
plot_confusion_matrix(cm,list(label.values()),title= f"CM Test Acc:{acc}")

 Surprising there is less underfitting problem but the accuracy as a whole is not good. Let's train it to 50 epochs

In [38]:
BS  = 64
epochs = 50
history2 = resnet_addsub.fit_generator(datagen_train1.flow(X_train1, Y_train1, batch_size=BS), 
                   steps_per_epoch=len(X_train1)//BS, # how many generators to go through per epoch
                   epochs=epochs, verbose=1,
                   validation_data=datagen_val1.flow(X_val1,Y_val1, batch_size=BS),
                   validation_steps=len(X_val1)//BS,
                   callbacks=[EarlyStopping(monitor='loss', patience=5)],
                  )
/users/sbataju/.conda/envs/tf_gpu/lib/python3.9/site-packages/keras/engine/training.py:1972: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
  warnings.warn('`Model.fit_generator` is deprecated and '
Epoch 1/50
140/140 [==============================] - 105s 752ms/step - loss: 1.5784 - accuracy: 0.4126 - val_loss: 1.2082 - val_accuracy: 0.6229
Epoch 2/50
140/140 [==============================] - 103s 733ms/step - loss: 1.5691 - accuracy: 0.4197 - val_loss: 1.7115 - val_accuracy: 0.3573
Epoch 3/50
140/140 [==============================] - 103s 736ms/step - loss: 1.5442 - accuracy: 0.4348 - val_loss: 2.6831 - val_accuracy: 0.3969
Epoch 4/50
140/140 [==============================] - 104s 739ms/step - loss: 1.5097 - accuracy: 0.4437 - val_loss: 1.7767 - val_accuracy: 0.4917
Epoch 5/50
140/140 [==============================] - 102s 729ms/step - loss: 1.5211 - accuracy: 0.4490 - val_loss: 1.4792 - val_accuracy: 0.4688
Epoch 6/50
140/140 [==============================] - 104s 745ms/step - loss: 1.5036 - accuracy: 0.4436 - val_loss: 2.5293 - val_accuracy: 0.3812
Epoch 7/50
140/140 [==============================] - 103s 736ms/step - loss: 1.4774 - accuracy: 0.4618 - val_loss: 1.7446 - val_accuracy: 0.4802
Epoch 8/50
140/140 [==============================] - 103s 731ms/step - loss: 1.4371 - accuracy: 0.4755 - val_loss: 1.2205 - val_accuracy: 0.6000
Epoch 9/50
140/140 [==============================] - 103s 734ms/step - loss: 1.4291 - accuracy: 0.4758 - val_loss: 1.1741 - val_accuracy: 0.6146
Epoch 10/50
140/140 [==============================] - 103s 733ms/step - loss: 1.3938 - accuracy: 0.4867 - val_loss: 1.0751 - val_accuracy: 0.6667
Epoch 11/50
140/140 [==============================] - 106s 756ms/step - loss: 1.4093 - accuracy: 0.4884 - val_loss: 1.0011 - val_accuracy: 0.7188
Epoch 12/50
140/140 [==============================] - 104s 743ms/step - loss: 1.3866 - accuracy: 0.4949 - val_loss: 1.2317 - val_accuracy: 0.5948
Epoch 13/50
140/140 [==============================] - 103s 735ms/step - loss: 1.3688 - accuracy: 0.5000 - val_loss: 1.0364 - val_accuracy: 0.6885
Epoch 14/50
140/140 [==============================] - 103s 737ms/step - loss: 1.3553 - accuracy: 0.5105 - val_loss: 1.4021 - val_accuracy: 0.5115
Epoch 15/50
140/140 [==============================] - 105s 746ms/step - loss: 1.3394 - accuracy: 0.5118 - val_loss: 1.1926 - val_accuracy: 0.6010
Epoch 16/50
140/140 [==============================] - 102s 727ms/step - loss: 1.3379 - accuracy: 0.5210 - val_loss: 1.0403 - val_accuracy: 0.6604
Epoch 17/50
140/140 [==============================] - 101s 722ms/step - loss: 1.3287 - accuracy: 0.5132 - val_loss: 1.0189 - val_accuracy: 0.6812
Epoch 18/50
140/140 [==============================] - 102s 728ms/step - loss: 1.3138 - accuracy: 0.5301 - val_loss: 0.9906 - val_accuracy: 0.6583
Epoch 19/50
140/140 [==============================] - 104s 739ms/step - loss: 1.3163 - accuracy: 0.5253 - val_loss: 0.9497 - val_accuracy: 0.7083
Epoch 20/50
140/140 [==============================] - 103s 738ms/step - loss: 1.3118 - accuracy: 0.5219 - val_loss: 0.8689 - val_accuracy: 0.7354
Epoch 21/50
140/140 [==============================] - 103s 731ms/step - loss: 1.2930 - accuracy: 0.5329 - val_loss: 1.0783 - val_accuracy: 0.6469
Epoch 22/50
140/140 [==============================] - 104s 741ms/step - loss: 1.3262 - accuracy: 0.5229 - val_loss: 0.8954 - val_accuracy: 0.7031
Epoch 23/50
140/140 [==============================] - 102s 731ms/step - loss: 1.2665 - accuracy: 0.5455 - val_loss: 0.9307 - val_accuracy: 0.7115
Epoch 24/50
140/140 [==============================] - 89s 634ms/step - loss: 1.2835 - accuracy: 0.5350 - val_loss: 0.9460 - val_accuracy: 0.6896
Epoch 25/50
140/140 [==============================] - 103s 736ms/step - loss: 1.2783 - accuracy: 0.5404 - val_loss: 0.8978 - val_accuracy: 0.7521
Epoch 26/50
140/140 [==============================] - 103s 734ms/step - loss: 1.2614 - accuracy: 0.5509 - val_loss: 0.8513 - val_accuracy: 0.7719
Epoch 27/50
140/140 [==============================] - 103s 735ms/step - loss: 1.2778 - accuracy: 0.5434 - val_loss: 0.9446 - val_accuracy: 0.6771
Epoch 28/50
140/140 [==============================] - 103s 737ms/step - loss: 1.2587 - accuracy: 0.5416 - val_loss: 1.1496 - val_accuracy: 0.6948
Epoch 29/50
140/140 [==============================] - 105s 752ms/step - loss: 1.2753 - accuracy: 0.5413 - val_loss: 0.8566 - val_accuracy: 0.7198
Epoch 30/50
140/140 [==============================] - 103s 736ms/step - loss: 1.2540 - accuracy: 0.5490 - val_loss: 0.8458 - val_accuracy: 0.7479
Epoch 31/50
140/140 [==============================] - 103s 732ms/step - loss: 1.2561 - accuracy: 0.5504 - val_loss: 0.7688 - val_accuracy: 0.7760
Epoch 32/50
140/140 [==============================] - 103s 737ms/step - loss: 1.2296 - accuracy: 0.5561 - val_loss: 0.8339 - val_accuracy: 0.7698
Epoch 33/50
140/140 [==============================] - 102s 729ms/step - loss: 1.2424 - accuracy: 0.5526 - val_loss: 1.7535 - val_accuracy: 0.5354
Epoch 34/50
140/140 [==============================] - 102s 727ms/step - loss: 1.2426 - accuracy: 0.5593 - val_loss: 0.9496 - val_accuracy: 0.6760
Epoch 35/50
140/140 [==============================] - 103s 733ms/step - loss: 1.2512 - accuracy: 0.5545 - val_loss: 0.8673 - val_accuracy: 0.7469
Epoch 36/50
140/140 [==============================] - 104s 745ms/step - loss: 1.2037 - accuracy: 0.5659 - val_loss: 0.8042 - val_accuracy: 0.7854
Epoch 37/50
140/140 [==============================] - 103s 738ms/step - loss: 1.2173 - accuracy: 0.5680 - val_loss: 0.7921 - val_accuracy: 0.7896
Epoch 38/50
140/140 [==============================] - 104s 739ms/step - loss: 1.2268 - accuracy: 0.5592 - val_loss: 0.7785 - val_accuracy: 0.7906
Epoch 39/50
140/140 [==============================] - 102s 731ms/step - loss: 1.2037 - accuracy: 0.5655 - val_loss: 0.8589 - val_accuracy: 0.7250
Epoch 40/50
140/140 [==============================] - 104s 740ms/step - loss: 1.2229 - accuracy: 0.5536 - val_loss: 0.9725 - val_accuracy: 0.7344
Epoch 41/50
140/140 [==============================] - 103s 738ms/step - loss: 1.2041 - accuracy: 0.5696 - val_loss: 0.7702 - val_accuracy: 0.7677
Epoch 42/50
140/140 [==============================] - 103s 737ms/step - loss: 1.2135 - accuracy: 0.5650 - val_loss: 0.8828 - val_accuracy: 0.7219
Epoch 43/50
140/140 [==============================] - 103s 736ms/step - loss: 1.2000 - accuracy: 0.5669 - val_loss: 0.7097 - val_accuracy: 0.8323
Epoch 44/50
140/140 [==============================] - 103s 738ms/step - loss: 1.1932 - accuracy: 0.5695 - val_loss: 0.8915 - val_accuracy: 0.7677
Epoch 45/50
140/140 [==============================] - 103s 735ms/step - loss: 1.2039 - accuracy: 0.5727 - val_loss: 1.8950 - val_accuracy: 0.5677
Epoch 46/50
140/140 [==============================] - 103s 733ms/step - loss: 1.1995 - accuracy: 0.5706 - val_loss: 1.1151 - val_accuracy: 0.6187
Epoch 47/50
140/140 [==============================] - 103s 732ms/step - loss: 1.1831 - accuracy: 0.5746 - val_loss: 0.7521 - val_accuracy: 0.7802
Epoch 48/50
140/140 [==============================] - 103s 736ms/step - loss: 1.1860 - accuracy: 0.5752 - val_loss: 0.6378 - val_accuracy: 0.8427
Epoch 49/50
140/140 [==============================] - 103s 738ms/step - loss: 1.1830 - accuracy: 0.5791 - val_loss: 1.0962 - val_accuracy: 0.6896
Epoch 50/50
140/140 [==============================] - 104s 739ms/step - loss: 1.1619 - accuracy: 0.5873 - val_loss: 1.1048 - val_accuracy: 0.6729
In [39]:
plot_history(history2)
yhat = np.argmax(resnet_addsub.predict(x_test1), axis=1)
acc = mt.accuracy_score(y_test1,yhat)
cm = mt.confusion_matrix(y_test1,yhat)
plot_confusion_matrix(cm,list(label.values()),title= f"CM Test Acc:{acc}")
Well, there is underfitting and the accuracy not improving at all, I will leaving this as is. 

Alexnet + Resnet

Modified Alexnet so that start of the last three convolution layer is added to the output as seen here.



In [106]:
fig = plt.figure(figsize=(60,6))
img = mpimg.imread('alex_resnet.png')
plt.imshow(img)

plt.show()
In [35]:
%%time
 
from tensorflow.keras.layers import Add, Input
from tensorflow.keras.layers import average, concatenate
from tensorflow.keras.models import Model
from tensorflow.keras.regularizers import l2
img_wh = 224
input_holder = Input(shape=(img_wh, img_wh, 3))

# start with a conv layer
# tf.keras.layers.Conv2D(filters=96, kernel_size=11, strides=4,
#                        activation='relu',input_shape=(224,224,3)),
x = Conv2D(filters=96,
               input_shape = (img_wh,img_wh,1),
               kernel_size=(11,11),
               strides=4,
               kernel_initializer='he_uniform', 
               kernel_regularizer=l2(l2_lambda),
               padding='same', 
               activation='relu', 
               data_format="channels_last")(input_holder)

#tf.keras.layers.MaxPool2D(pool_size=3, strides=2),
x = MaxPooling2D(pool_size=(3, 3),strides=2, data_format="channels_last")(x)

#tf.keras.layers.Conv2D(filters=256, kernel_size=5, padding='same',activation='relu')
x = Conv2D(filters=256,
               kernel_size=(5,5),
               kernel_initializer='he_uniform', 
               kernel_regularizer=l2(l2_lambda),
               padding='same', 
               activation='relu', 
               data_format="channels_last")(x)
#tf.keras.layers.MaxPool2D(pool_size=3, strides=2),
x_split = MaxPooling2D(pool_size=(3, 3), strides=2, data_format="channels_last")(x)

# tf.keras.layers.Conv2D(filters=384, kernel_size=3, padding='same',activation='relu'),
# tf.keras.layers.Conv2D(filters=384, kernel_size=3, padding='same',activation='relu'),
# tf.keras.layers.Conv2D(filters=256, kernel_size=3, padding='same',activation='relu'),

x = Conv2D(filters=384,
               kernel_size=(3,3),
               kernel_initializer='he_uniform', 
               kernel_regularizer=l2(l2_lambda),
               padding='same', 
               activation='relu', 
               data_format="channels_last")(x_split)

x = Conv2D(filters=384,
               kernel_size=(3,3),
               kernel_initializer='he_uniform', 
               kernel_regularizer=l2(l2_lambda),
               padding='same', 
               activation='relu', 
               data_format="channels_last")(x)

x = Conv2D(filters=256,
               kernel_size=(3,3),
               kernel_initializer='he_uniform', 
               kernel_regularizer=l2(l2_lambda),
               padding='same', 
               activation='relu', 
               data_format="channels_last")(x)

# now add back in the split layer, x_split (residual added in)
x = Add()([x, x_split])
x = Activation("relu")(x)
#tf.keras.layers.MaxPool2D(pool_size=3, strides=2),
x = MaxPooling2D(pool_size=(3, 3),strides=2, data_format="channels_last")(x)
# tf.keras.layers.Flatten(),
# tf.keras.layers.Dense(4096, activation='relu'),
# tf.keras.layers.Dropout(0.5),
# tf.keras.layers.Dense(4096, activation='relu'),
# tf.keras.layers.Dropout(0.5),
# tf.keras.layers.Dense(num_classes)])

x = Flatten()(x)
x = Dropout(0.25)(x)
x = Dense(4096)(x)
x = Activation("relu")(x)
x = Dropout(0.5)(x)
x = Dense(4096)(x)
x = Activation("relu")(x)
x = Dropout(0.5)(x)
x = Dense(NUM_CLASSES)(x)
x = Activation('softmax')(x)

alex_resnet = Model(inputs=input_holder,outputs=x)

alex_resnet.summary()
Model: "model_2"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_5 (InputLayer)            [(None, 224, 224, 3) 0                                            
__________________________________________________________________________________________________
conv2d_21 (Conv2D)              (None, 56, 56, 96)   34944       input_5[0][0]                    
__________________________________________________________________________________________________
max_pooling2d_8 (MaxPooling2D)  (None, 27, 27, 96)   0           conv2d_21[0][0]                  
__________________________________________________________________________________________________
conv2d_22 (Conv2D)              (None, 27, 27, 256)  614656      max_pooling2d_8[0][0]            
__________________________________________________________________________________________________
max_pooling2d_9 (MaxPooling2D)  (None, 13, 13, 256)  0           conv2d_22[0][0]                  
__________________________________________________________________________________________________
conv2d_23 (Conv2D)              (None, 13, 13, 384)  885120      max_pooling2d_9[0][0]            
__________________________________________________________________________________________________
conv2d_24 (Conv2D)              (None, 13, 13, 384)  1327488     conv2d_23[0][0]                  
__________________________________________________________________________________________________
conv2d_25 (Conv2D)              (None, 13, 13, 256)  884992      conv2d_24[0][0]                  
__________________________________________________________________________________________________
add_3 (Add)                     (None, 13, 13, 256)  0           conv2d_25[0][0]                  
                                                                 max_pooling2d_9[0][0]            
__________________________________________________________________________________________________
activation_7 (Activation)       (None, 13, 13, 256)  0           add_3[0][0]                      
__________________________________________________________________________________________________
max_pooling2d_10 (MaxPooling2D) (None, 6, 6, 256)    0           activation_7[0][0]               
__________________________________________________________________________________________________
flatten_2 (Flatten)             (None, 9216)         0           max_pooling2d_10[0][0]           
__________________________________________________________________________________________________
dropout_5 (Dropout)             (None, 9216)         0           flatten_2[0][0]                  
__________________________________________________________________________________________________
dense_5 (Dense)                 (None, 4096)         37752832    dropout_5[0][0]                  
__________________________________________________________________________________________________
activation_8 (Activation)       (None, 4096)         0           dense_5[0][0]                    
__________________________________________________________________________________________________
dropout_6 (Dropout)             (None, 4096)         0           activation_8[0][0]               
__________________________________________________________________________________________________
dense_6 (Dense)                 (None, 4096)         16781312    dropout_6[0][0]                  
__________________________________________________________________________________________________
activation_9 (Activation)       (None, 4096)         0           dense_6[0][0]                    
__________________________________________________________________________________________________
dropout_7 (Dropout)             (None, 4096)         0           activation_9[0][0]               
__________________________________________________________________________________________________
dense_7 (Dense)                 (None, 10)           40970       dropout_7[0][0]                  
__________________________________________________________________________________________________
activation_10 (Activation)      (None, 10)           0           dense_7[0][0]                    
==================================================================================================
Total params: 58,322,314
Trainable params: 58,322,314
Non-trainable params: 0
__________________________________________________________________________________________________
CPU times: user 119 ms, sys: 2.18 ms, total: 121 ms
Wall time: 119 ms
In [37]:
alex_resnet.compile(loss='categorical_crossentropy', # 'categorical_crossentropy' 'mean_squared_error'
                optimizer='adam', # 'adadelta' 'rmsprop'
                metrics=['accuracy'])
In [38]:
BS  = 128
epochs = 50
history = alex_resnet.fit_generator(datagen_train1.flow(X_train1, Y_train1, batch_size=BS), 
                   steps_per_epoch=len(X_train)//BS, # how many generators to go through per epoch
                   epochs=epochs, verbose=1,
                   validation_data=datagen_val1.flow(X_val1,Y_val1, batch_size=BS),
                   validation_steps=len(X_val)//BS,
                   callbacks=[EarlyStopping(monitor='val_loss', patience=5)],
                  )
/users/sbataju/.conda/envs/tf_gpu/lib/python3.9/site-packages/keras/engine/training.py:1972: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
  warnings.warn('`Model.fit_generator` is deprecated and '
2022-12-03 12:26:55.546684: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:185] None of the MLIR Optimization Passes are enabled (registered 2)
Epoch 1/50
2022-12-03 12:27:04.433365: I tensorflow/stream_executor/cuda/cuda_dnn.cc:369] Loaded cuDNN version 8401
70/70 [==============================] - 151s 2s/step - loss: 3.4883 - accuracy: 0.1125 - val_loss: 2.4641 - val_accuracy: 0.1384
Epoch 2/50
70/70 [==============================] - 125s 2s/step - loss: 2.4671 - accuracy: 0.1362 - val_loss: 2.4005 - val_accuracy: 0.1864
Epoch 3/50
70/70 [==============================] - 127s 2s/step - loss: 2.4105 - accuracy: 0.1642 - val_loss: 2.3134 - val_accuracy: 0.2165
Epoch 4/50
70/70 [==============================] - 126s 2s/step - loss: 2.2777 - accuracy: 0.2104 - val_loss: 2.1101 - val_accuracy: 0.3013
Epoch 5/50
70/70 [==============================] - 126s 2s/step - loss: 2.0915 - accuracy: 0.2867 - val_loss: 1.7956 - val_accuracy: 0.3806
Epoch 6/50
70/70 [==============================] - 127s 2s/step - loss: 2.0317 - accuracy: 0.3496 - val_loss: 1.4561 - val_accuracy: 0.5480
Epoch 7/50
70/70 [==============================] - 127s 2s/step - loss: 1.4686 - accuracy: 0.5238 - val_loss: 1.1323 - val_accuracy: 0.6339
Epoch 8/50
70/70 [==============================] - 127s 2s/step - loss: 1.2343 - accuracy: 0.6003 - val_loss: 1.0358 - val_accuracy: 0.6518
Epoch 9/50
70/70 [==============================] - 127s 2s/step - loss: 1.1103 - accuracy: 0.6480 - val_loss: 0.8941 - val_accuracy: 0.7076
Epoch 10/50
70/70 [==============================] - 127s 2s/step - loss: 0.9600 - accuracy: 0.6991 - val_loss: 0.9525 - val_accuracy: 0.7143
Epoch 11/50
70/70 [==============================] - 127s 2s/step - loss: 0.8394 - accuracy: 0.7403 - val_loss: 0.6460 - val_accuracy: 0.7958
Epoch 12/50
70/70 [==============================] - 126s 2s/step - loss: 0.7789 - accuracy: 0.7662 - val_loss: 0.7891 - val_accuracy: 0.7567
Epoch 13/50
70/70 [==============================] - 126s 2s/step - loss: 0.7267 - accuracy: 0.7790 - val_loss: 0.5520 - val_accuracy: 0.8359
Epoch 14/50
70/70 [==============================] - 127s 2s/step - loss: 0.7287 - accuracy: 0.7765 - val_loss: 0.5537 - val_accuracy: 0.8538
Epoch 15/50
70/70 [==============================] - 127s 2s/step - loss: 0.6563 - accuracy: 0.8047 - val_loss: 0.5245 - val_accuracy: 0.8549
Epoch 16/50
70/70 [==============================] - 127s 2s/step - loss: 0.5802 - accuracy: 0.8245 - val_loss: 0.4294 - val_accuracy: 0.8917
Epoch 17/50
70/70 [==============================] - 127s 2s/step - loss: 0.5521 - accuracy: 0.8362 - val_loss: 0.4334 - val_accuracy: 0.8783
Epoch 18/50
70/70 [==============================] - 127s 2s/step - loss: 0.5387 - accuracy: 0.8396 - val_loss: 0.4254 - val_accuracy: 0.8884
Epoch 19/50
70/70 [==============================] - 127s 2s/step - loss: 0.5182 - accuracy: 0.8456 - val_loss: 0.4181 - val_accuracy: 0.8828
Epoch 20/50
70/70 [==============================] - 127s 2s/step - loss: 0.4818 - accuracy: 0.8548 - val_loss: 0.3868 - val_accuracy: 0.9074
Epoch 21/50
70/70 [==============================] - 127s 2s/step - loss: 0.4901 - accuracy: 0.8596 - val_loss: 0.4588 - val_accuracy: 0.8795
Epoch 22/50
70/70 [==============================] - 126s 2s/step - loss: 0.4788 - accuracy: 0.8573 - val_loss: 0.3478 - val_accuracy: 0.9085
Epoch 23/50
70/70 [==============================] - 127s 2s/step - loss: 0.4453 - accuracy: 0.8706 - val_loss: 0.4418 - val_accuracy: 0.8839
Epoch 24/50
70/70 [==============================] - 127s 2s/step - loss: 0.4330 - accuracy: 0.8751 - val_loss: 0.3774 - val_accuracy: 0.9051
Epoch 25/50
70/70 [==============================] - 127s 2s/step - loss: 0.4023 - accuracy: 0.8842 - val_loss: 0.3207 - val_accuracy: 0.9163
Epoch 26/50
70/70 [==============================] - 127s 2s/step - loss: 0.3914 - accuracy: 0.8886 - val_loss: 0.3251 - val_accuracy: 0.9230
Epoch 27/50
70/70 [==============================] - 127s 2s/step - loss: 0.4300 - accuracy: 0.8720 - val_loss: 0.3500 - val_accuracy: 0.9051
Epoch 28/50
70/70 [==============================] - 127s 2s/step - loss: 0.3779 - accuracy: 0.8957 - val_loss: 0.3598 - val_accuracy: 0.8996
Epoch 29/50
70/70 [==============================] - 127s 2s/step - loss: 0.4541 - accuracy: 0.8680 - val_loss: 0.3103 - val_accuracy: 0.8984
Epoch 30/50
70/70 [==============================] - 127s 2s/step - loss: 0.3978 - accuracy: 0.8855 - val_loss: 0.2992 - val_accuracy: 0.9185
Epoch 31/50
70/70 [==============================] - 128s 2s/step - loss: 0.3828 - accuracy: 0.8913 - val_loss: 0.3084 - val_accuracy: 0.9196
Epoch 32/50
70/70 [==============================] - 127s 2s/step - loss: 0.3423 - accuracy: 0.9027 - val_loss: 0.3041 - val_accuracy: 0.9163
Epoch 33/50
70/70 [==============================] - 127s 2s/step - loss: 0.3891 - accuracy: 0.8862 - val_loss: 0.2852 - val_accuracy: 0.9230
Epoch 34/50
70/70 [==============================] - 127s 2s/step - loss: 0.3272 - accuracy: 0.9084 - val_loss: 0.2844 - val_accuracy: 0.9141
Epoch 35/50
70/70 [==============================] - 126s 2s/step - loss: 0.3502 - accuracy: 0.8970 - val_loss: 0.4191 - val_accuracy: 0.8817
Epoch 36/50
70/70 [==============================] - 125s 2s/step - loss: 0.3463 - accuracy: 0.9018 - val_loss: 0.2913 - val_accuracy: 0.9330
Epoch 37/50
70/70 [==============================] - 126s 2s/step - loss: 0.3214 - accuracy: 0.9101 - val_loss: 0.3362 - val_accuracy: 0.9107
Epoch 38/50
70/70 [==============================] - 127s 2s/step - loss: 0.2983 - accuracy: 0.9181 - val_loss: 0.2813 - val_accuracy: 0.9163
Epoch 39/50
70/70 [==============================] - 126s 2s/step - loss: 0.3105 - accuracy: 0.9121 - val_loss: 0.3074 - val_accuracy: 0.9174
Epoch 40/50
70/70 [==============================] - 127s 2s/step - loss: 0.2990 - accuracy: 0.9152 - val_loss: 0.2474 - val_accuracy: 0.9364
Epoch 41/50
70/70 [==============================] - 126s 2s/step - loss: 0.2875 - accuracy: 0.9179 - val_loss: 0.3100 - val_accuracy: 0.9174
Epoch 42/50
70/70 [==============================] - 126s 2s/step - loss: 0.2865 - accuracy: 0.9181 - val_loss: 0.2297 - val_accuracy: 0.9420
Epoch 43/50
70/70 [==============================] - 127s 2s/step - loss: 0.2891 - accuracy: 0.9201 - val_loss: 0.2613 - val_accuracy: 0.9275
Epoch 44/50
70/70 [==============================] - 127s 2s/step - loss: 0.2878 - accuracy: 0.9196 - val_loss: 0.2623 - val_accuracy: 0.9364
Epoch 45/50
70/70 [==============================] - 127s 2s/step - loss: 0.2978 - accuracy: 0.9175 - val_loss: 0.2581 - val_accuracy: 0.9286
Epoch 46/50
70/70 [==============================] - 127s 2s/step - loss: 0.2870 - accuracy: 0.9159 - val_loss: 0.2382 - val_accuracy: 0.9330
Epoch 47/50
70/70 [==============================] - 126s 2s/step - loss: 0.2905 - accuracy: 0.9169 - val_loss: 0.2398 - val_accuracy: 0.9342
In [62]:
plot_history(history)
yhat = np.argmax(alex_resnet.predict(x_test1), axis=1)
acc = mt.accuracy_score(y_test1,yhat)
cm = mt.confusion_matrix(y_test1,yhat)
plot_confusion_matrix(cm,list(label.values()),title= f"CM Test Acc:{acc:0.3f}")
In [63]:
# X_train1, Y_train1,
yhat = np.argmax(alex_resnet.predict(X_train1), axis=1)
acc = mt.accuracy_score(np.argmax(Y_train1, axis=1),yhat)
cm = mt.confusion_matrix(np.argmax(Y_train1, axis=1),yhat)
plot_confusion_matrix(cm,list(label.values()),title= f"CM Train Acc:{acc:0.3f}")
In [64]:
# X_train1, Y_train1,
yhat = np.argmax(alex_resnet.predict(X_val1), axis=1)
acc = mt.accuracy_score(np.argmax(Y_val1, axis=1),yhat)
cm = mt.confusion_matrix(np.argmax(Y_val1, axis=1),yhat)
plot_confusion_matrix(cm,list(label.values()),title= f"CM Val Acc:{acc:0.3f}")
This network is the best performing of all the CNN I have tried. I don't see underfitting or overfitting and the network is converging to high accuracy greater than ~90%


Transfer Learning

https://stackoverflow.com/questions/51336761/how-to-do-transfer-learning-on-our-own-models

Showing trasfer learing in the AlexNet + ResNet I have above. First, save and load the model.

In [69]:
alex_resnet.save('alex_resnet.h5')
/users/sbataju/.conda/envs/tf_gpu/lib/python3.9/site-packages/keras/utils/generic_utils.py:494: CustomMaskWarning: Custom mask layers require a config and must override get_config. When loading, the custom mask layer must be passed to the custom_objects argument.
  warnings.warn('Custom mask layers require a config and must override '
In [66]:
# Recreate the exact same model, including its weights and the optimizer
new_model_alex_resnet = tf.keras.models.load_model('alex_resnet.h5')

# Show the model architecture
new_model_alex_resnet.summary()
Model: "model_2"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_5 (InputLayer)            [(None, 224, 224, 3) 0                                            
__________________________________________________________________________________________________
conv2d_21 (Conv2D)              (None, 56, 56, 96)   34944       input_5[0][0]                    
__________________________________________________________________________________________________
max_pooling2d_8 (MaxPooling2D)  (None, 27, 27, 96)   0           conv2d_21[0][0]                  
__________________________________________________________________________________________________
conv2d_22 (Conv2D)              (None, 27, 27, 256)  614656      max_pooling2d_8[0][0]            
__________________________________________________________________________________________________
max_pooling2d_9 (MaxPooling2D)  (None, 13, 13, 256)  0           conv2d_22[0][0]                  
__________________________________________________________________________________________________
conv2d_23 (Conv2D)              (None, 13, 13, 384)  885120      max_pooling2d_9[0][0]            
__________________________________________________________________________________________________
conv2d_24 (Conv2D)              (None, 13, 13, 384)  1327488     conv2d_23[0][0]                  
__________________________________________________________________________________________________
conv2d_25 (Conv2D)              (None, 13, 13, 256)  884992      conv2d_24[0][0]                  
__________________________________________________________________________________________________
add_3 (Add)                     (None, 13, 13, 256)  0           conv2d_25[0][0]                  
                                                                 max_pooling2d_9[0][0]            
__________________________________________________________________________________________________
activation_7 (Activation)       (None, 13, 13, 256)  0           add_3[0][0]                      
__________________________________________________________________________________________________
max_pooling2d_10 (MaxPooling2D) (None, 6, 6, 256)    0           activation_7[0][0]               
__________________________________________________________________________________________________
flatten_2 (Flatten)             (None, 9216)         0           max_pooling2d_10[0][0]           
__________________________________________________________________________________________________
dropout_5 (Dropout)             (None, 9216)         0           flatten_2[0][0]                  
__________________________________________________________________________________________________
dense_5 (Dense)                 (None, 4096)         37752832    dropout_5[0][0]                  
__________________________________________________________________________________________________
activation_8 (Activation)       (None, 4096)         0           dense_5[0][0]                    
__________________________________________________________________________________________________
dropout_6 (Dropout)             (None, 4096)         0           activation_8[0][0]               
__________________________________________________________________________________________________
dense_6 (Dense)                 (None, 4096)         16781312    dropout_6[0][0]                  
__________________________________________________________________________________________________
activation_9 (Activation)       (None, 4096)         0           dense_6[0][0]                    
__________________________________________________________________________________________________
dropout_7 (Dropout)             (None, 4096)         0           activation_9[0][0]               
__________________________________________________________________________________________________
dense_7 (Dense)                 (None, 10)           40970       dropout_7[0][0]                  
__________________________________________________________________________________________________
activation_10 (Activation)      (None, 10)           0           dense_7[0][0]                    
==================================================================================================
Total params: 58,322,314
Trainable params: 58,322,314
Non-trainable params: 0
__________________________________________________________________________________________________
In [67]:
yhat = np.argmax(new_model_alex_resnet.predict(x_test1), axis=1)
acc = mt.accuracy_score(y_test1,yhat)
cm = mt.confusion_matrix(y_test1,yhat)
plot_confusion_matrix(cm,list(label.values()),title= f"CM Test Acc:{acc:0.3f}")

Change the bottom layer

Testing the loaded model, we see it give the same accuracy as the model we had tested before. Now, freezing the weights for the convolution layers and add dense layer with changed nodes from 4096 to 1024 and enable learning in the bottom layer.



In [107]:
fig = plt.figure(figsize=(60,6))
img = mpimg.imread('new_model.png')
plt.imshow(img)

plt.show()
In [68]:
print("Number of layers in the base model: ", len(new_model_alex_resnet.layers))
Number of layers in the base model:  21
In [69]:
# Fine-tune from this layer onwards
fine_tune_at = 11

# Freeze all the layers before the `fine_tune_at` layer
for layer in new_model_alex_resnet.layers[:fine_tune_at]:
    layer.trainable = False
for layer in new_model_alex_resnet.layers[fine_tune_at:]:
    layer.trainable = True
In [70]:
new_model_alex_resnet.summary()
Model: "model_2"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_5 (InputLayer)            [(None, 224, 224, 3) 0                                            
__________________________________________________________________________________________________
conv2d_21 (Conv2D)              (None, 56, 56, 96)   34944       input_5[0][0]                    
__________________________________________________________________________________________________
max_pooling2d_8 (MaxPooling2D)  (None, 27, 27, 96)   0           conv2d_21[0][0]                  
__________________________________________________________________________________________________
conv2d_22 (Conv2D)              (None, 27, 27, 256)  614656      max_pooling2d_8[0][0]            
__________________________________________________________________________________________________
max_pooling2d_9 (MaxPooling2D)  (None, 13, 13, 256)  0           conv2d_22[0][0]                  
__________________________________________________________________________________________________
conv2d_23 (Conv2D)              (None, 13, 13, 384)  885120      max_pooling2d_9[0][0]            
__________________________________________________________________________________________________
conv2d_24 (Conv2D)              (None, 13, 13, 384)  1327488     conv2d_23[0][0]                  
__________________________________________________________________________________________________
conv2d_25 (Conv2D)              (None, 13, 13, 256)  884992      conv2d_24[0][0]                  
__________________________________________________________________________________________________
add_3 (Add)                     (None, 13, 13, 256)  0           conv2d_25[0][0]                  
                                                                 max_pooling2d_9[0][0]            
__________________________________________________________________________________________________
activation_7 (Activation)       (None, 13, 13, 256)  0           add_3[0][0]                      
__________________________________________________________________________________________________
max_pooling2d_10 (MaxPooling2D) (None, 6, 6, 256)    0           activation_7[0][0]               
__________________________________________________________________________________________________
flatten_2 (Flatten)             (None, 9216)         0           max_pooling2d_10[0][0]           
__________________________________________________________________________________________________
dropout_5 (Dropout)             (None, 9216)         0           flatten_2[0][0]                  
__________________________________________________________________________________________________
dense_5 (Dense)                 (None, 4096)         37752832    dropout_5[0][0]                  
__________________________________________________________________________________________________
activation_8 (Activation)       (None, 4096)         0           dense_5[0][0]                    
__________________________________________________________________________________________________
dropout_6 (Dropout)             (None, 4096)         0           activation_8[0][0]               
__________________________________________________________________________________________________
dense_6 (Dense)                 (None, 4096)         16781312    dropout_6[0][0]                  
__________________________________________________________________________________________________
activation_9 (Activation)       (None, 4096)         0           dense_6[0][0]                    
__________________________________________________________________________________________________
dropout_7 (Dropout)             (None, 4096)         0           activation_9[0][0]               
__________________________________________________________________________________________________
dense_7 (Dense)                 (None, 10)           40970       dropout_7[0][0]                  
__________________________________________________________________________________________________
activation_10 (Activation)      (None, 10)           0           dense_7[0][0]                    
==================================================================================================
Total params: 58,322,314
Trainable params: 54,575,114
Non-trainable params: 3,747,200
__________________________________________________________________________________________________
In [74]:
add_x =  new_model_alex_resnet.layers[fine_tune_at].output
add_x = Dropout(0.25)(add_x)
add_x = Dense(1024)(add_x)
add_x = Activation("relu")(add_x)
add_x = Dropout(0.5)(add_x)
add_x = Dense(NUM_CLASSES)(add_x)
add_x = Activation('softmax')(add_x)
new_model = Model(inputs=new_model_alex_resnet.input,outputs=add_x)
In [75]:
new_model.summary()
Model: "model_3"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_5 (InputLayer)            [(None, 224, 224, 3) 0                                            
__________________________________________________________________________________________________
conv2d_21 (Conv2D)              (None, 56, 56, 96)   34944       input_5[0][0]                    
__________________________________________________________________________________________________
max_pooling2d_8 (MaxPooling2D)  (None, 27, 27, 96)   0           conv2d_21[0][0]                  
__________________________________________________________________________________________________
conv2d_22 (Conv2D)              (None, 27, 27, 256)  614656      max_pooling2d_8[0][0]            
__________________________________________________________________________________________________
max_pooling2d_9 (MaxPooling2D)  (None, 13, 13, 256)  0           conv2d_22[0][0]                  
__________________________________________________________________________________________________
conv2d_23 (Conv2D)              (None, 13, 13, 384)  885120      max_pooling2d_9[0][0]            
__________________________________________________________________________________________________
conv2d_24 (Conv2D)              (None, 13, 13, 384)  1327488     conv2d_23[0][0]                  
__________________________________________________________________________________________________
conv2d_25 (Conv2D)              (None, 13, 13, 256)  884992      conv2d_24[0][0]                  
__________________________________________________________________________________________________
add_3 (Add)                     (None, 13, 13, 256)  0           conv2d_25[0][0]                  
                                                                 max_pooling2d_9[0][0]            
__________________________________________________________________________________________________
activation_7 (Activation)       (None, 13, 13, 256)  0           add_3[0][0]                      
__________________________________________________________________________________________________
max_pooling2d_10 (MaxPooling2D) (None, 6, 6, 256)    0           activation_7[0][0]               
__________________________________________________________________________________________________
flatten_2 (Flatten)             (None, 9216)         0           max_pooling2d_10[0][0]           
__________________________________________________________________________________________________
dropout_16 (Dropout)            (None, 9216)         0           flatten_2[0][0]                  
__________________________________________________________________________________________________
dense_16 (Dense)                (None, 1024)         9438208     dropout_16[0][0]                 
__________________________________________________________________________________________________
activation_8 (Activation)       (None, 1024)         0           dense_16[0][0]                   
__________________________________________________________________________________________________
dropout_17 (Dropout)            (None, 1024)         0           activation_8[0][0]               
__________________________________________________________________________________________________
dense_17 (Dense)                (None, 10)           10250       dropout_17[0][0]                 
__________________________________________________________________________________________________
activation_9 (Activation)       (None, 10)           0           dense_17[0][0]                   
==================================================================================================
Total params: 13,195,658
Trainable params: 9,448,458
Non-trainable params: 3,747,200
__________________________________________________________________________________________________
In [80]:
# https://www.tensorflow.org/tutorials/images/transfer_learning#compile_the_model_2
#Decrease the learning rate.
new_model.compile(loss='categorical_crossentropy', # 'categorical_crossentropy' 'mean_squared_error'
                optimizer=tf.keras.optimizers.Adam(learning_rate=0.001/10), # 'adadelta' 'rmsprop'
                metrics=['accuracy'])
In [81]:
BS  = 128
epochs = 25
history1 = new_model.fit_generator(datagen_train1.flow(X_train1, Y_train1, batch_size=BS), 
                   steps_per_epoch=len(X_train)//BS, # how many generators to go through per epoch
                   epochs=epochs, verbose=1,
                   validation_data=datagen_val1.flow(X_val1,Y_val1, batch_size=BS),
                   validation_steps=len(X_val)//BS,
                   callbacks=[EarlyStopping(monitor='val_loss', patience=5)],
                  )
/users/sbataju/.conda/envs/tf_gpu/lib/python3.9/site-packages/keras/engine/training.py:1972: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
  warnings.warn('`Model.fit_generator` is deprecated and '
Epoch 1/25
70/70 [==============================] - 129s 2s/step - loss: 1.9132 - accuracy: 0.5424 - val_loss: 1.4589 - val_accuracy: 0.6797
Epoch 2/25
70/70 [==============================] - 127s 2s/step - loss: 1.2361 - accuracy: 0.7232 - val_loss: 0.9934 - val_accuracy: 0.7879
Epoch 3/25
70/70 [==============================] - 127s 2s/step - loss: 0.9292 - accuracy: 0.7823 - val_loss: 0.7816 - val_accuracy: 0.8270
Epoch 4/25
70/70 [==============================] - 126s 2s/step - loss: 0.7627 - accuracy: 0.8233 - val_loss: 0.6734 - val_accuracy: 0.8460
Epoch 5/25
70/70 [==============================] - 125s 2s/step - loss: 0.6782 - accuracy: 0.8327 - val_loss: 0.5841 - val_accuracy: 0.8605
Epoch 6/25
70/70 [==============================] - 126s 2s/step - loss: 0.6066 - accuracy: 0.8530 - val_loss: 0.5275 - val_accuracy: 0.8750
Epoch 7/25
70/70 [==============================] - 126s 2s/step - loss: 0.5625 - accuracy: 0.8588 - val_loss: 0.4918 - val_accuracy: 0.8761
Epoch 8/25
70/70 [==============================] - 127s 2s/step - loss: 0.5336 - accuracy: 0.8615 - val_loss: 0.4592 - val_accuracy: 0.8739
Epoch 9/25
70/70 [==============================] - 127s 2s/step - loss: 0.5067 - accuracy: 0.8670 - val_loss: 0.4379 - val_accuracy: 0.8873
Epoch 10/25
70/70 [==============================] - 127s 2s/step - loss: 0.4806 - accuracy: 0.8759 - val_loss: 0.4165 - val_accuracy: 0.8917
Epoch 11/25
70/70 [==============================] - 127s 2s/step - loss: 0.4700 - accuracy: 0.8757 - val_loss: 0.4221 - val_accuracy: 0.8984
Epoch 12/25
70/70 [==============================] - 126s 2s/step - loss: 0.4441 - accuracy: 0.8851 - val_loss: 0.3927 - val_accuracy: 0.8951
Epoch 13/25
70/70 [==============================] - 127s 2s/step - loss: 0.4342 - accuracy: 0.8835 - val_loss: 0.3854 - val_accuracy: 0.8917
Epoch 14/25
70/70 [==============================] - 126s 2s/step - loss: 0.4194 - accuracy: 0.8876 - val_loss: 0.3666 - val_accuracy: 0.8962
Epoch 15/25
70/70 [==============================] - 127s 2s/step - loss: 0.4077 - accuracy: 0.8915 - val_loss: 0.3714 - val_accuracy: 0.9107
Epoch 16/25
70/70 [==============================] - 126s 2s/step - loss: 0.4061 - accuracy: 0.8889 - val_loss: 0.3534 - val_accuracy: 0.9118
Epoch 17/25
70/70 [==============================] - 127s 2s/step - loss: 0.4010 - accuracy: 0.8895 - val_loss: 0.3460 - val_accuracy: 0.9196
Epoch 18/25
70/70 [==============================] - 126s 2s/step - loss: 0.3826 - accuracy: 0.8968 - val_loss: 0.3333 - val_accuracy: 0.9085
Epoch 19/25
70/70 [==============================] - 126s 2s/step - loss: 0.3824 - accuracy: 0.8956 - val_loss: 0.3293 - val_accuracy: 0.9141
Epoch 20/25
70/70 [==============================] - 126s 2s/step - loss: 0.3725 - accuracy: 0.8973 - val_loss: 0.3344 - val_accuracy: 0.9230
Epoch 21/25
70/70 [==============================] - 127s 2s/step - loss: 0.3624 - accuracy: 0.9033 - val_loss: 0.3245 - val_accuracy: 0.9129
Epoch 22/25
70/70 [==============================] - 127s 2s/step - loss: 0.3644 - accuracy: 0.9052 - val_loss: 0.3156 - val_accuracy: 0.9185
Epoch 23/25
70/70 [==============================] - 126s 2s/step - loss: 0.3550 - accuracy: 0.9044 - val_loss: 0.3165 - val_accuracy: 0.9174
Epoch 24/25
70/70 [==============================] - 126s 2s/step - loss: 0.3598 - accuracy: 0.9018 - val_loss: 0.3142 - val_accuracy: 0.9129
Epoch 25/25
70/70 [==============================] - 127s 2s/step - loss: 0.3357 - accuracy: 0.9099 - val_loss: 0.3116 - val_accuracy: 0.9118
In [82]:
plot_history(history1)
yhat = np.argmax(new_model.predict(x_test1), axis=1)
acc = mt.accuracy_score(y_test1,yhat)
cm = mt.confusion_matrix(y_test1,yhat)
plot_confusion_matrix(cm,list(label.values()),title= f"CM Test Acc:{acc:0.3f}")
In [83]:
# X_train1, Y_train1,
yhat = np.argmax(new_model.predict(X_train1), axis=1)
acc = mt.accuracy_score(np.argmax(Y_train1, axis=1),yhat)
cm = mt.confusion_matrix(np.argmax(Y_train1, axis=1),yhat)
plot_confusion_matrix(cm,list(label.values()),title= f"CM Train Acc:{acc:0.3f}")
In [86]:
# X_train1, Y_train1,
yhat = np.argmax(new_model.predict(X_val1), axis=1)
acc = mt.accuracy_score(np.argmax(Y_val1, axis=1),yhat)
cm = mt.confusion_matrix(np.argmax(Y_val1, axis=1),yhat)
plot_confusion_matrix(cm,list(label.values()),title= f"CM Val Acc:{acc:0.3f}")
Pretty nice results. Compared to the original network the result is very similar, just slight decrease (~2%). 

ResNet152V2

The ResNet152V2 is very large to plot efficively here. Here I am testing this network, only taking the top half and adding one dense layer.

In [114]:
fig = plt.figure(figsize=(60,10))
img = mpimg.imread('model_ResNet.png')
plt.imshow(img)

plt.show()
In [117]:
from tensorflow.keras.applications import ResNet152V2
from tensorflow.keras.layers import GlobalAveragePooling2D
In [120]:
def get_ResNet152V2():
    
    base_model = ResNet152V2(input_shape=(224,224,3), include_top=False)
    
    for layers in base_model.layers[:140]:
        layers.trainable = False
    for layers in base_model.layers[140:]:
        layers.trainable = True
        
    
    x = base_model.output
    x = GlobalAveragePooling2D()(x)
    x = Dense(1000, activation='relu')(x)
    pred = Dense(10, activation='softmax')(x)
    
    model = Model(inputs=base_model.input, outputs=pred)
    
    return model     
    
In [121]:
model = get_ResNet152V2()
model.summary()
Model: "model_6"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_8 (InputLayer)            [(None, 224, 224, 3) 0                                            
__________________________________________________________________________________________________
conv1_pad (ZeroPadding2D)       (None, 230, 230, 3)  0           input_8[0][0]                    
__________________________________________________________________________________________________
conv1_conv (Conv2D)             (None, 112, 112, 64) 9472        conv1_pad[0][0]                  
__________________________________________________________________________________________________
pool1_pad (ZeroPadding2D)       (None, 114, 114, 64) 0           conv1_conv[0][0]                 
__________________________________________________________________________________________________
pool1_pool (MaxPooling2D)       (None, 56, 56, 64)   0           pool1_pad[0][0]                  
__________________________________________________________________________________________________
conv2_block1_preact_bn (BatchNo (None, 56, 56, 64)   256         pool1_pool[0][0]                 
__________________________________________________________________________________________________
conv2_block1_preact_relu (Activ (None, 56, 56, 64)   0           conv2_block1_preact_bn[0][0]     
__________________________________________________________________________________________________
conv2_block1_1_conv (Conv2D)    (None, 56, 56, 64)   4096        conv2_block1_preact_relu[0][0]   
__________________________________________________________________________________________________
conv2_block1_1_bn (BatchNormali (None, 56, 56, 64)   256         conv2_block1_1_conv[0][0]        
__________________________________________________________________________________________________
conv2_block1_1_relu (Activation (None, 56, 56, 64)   0           conv2_block1_1_bn[0][0]          
__________________________________________________________________________________________________
conv2_block1_2_pad (ZeroPadding (None, 58, 58, 64)   0           conv2_block1_1_relu[0][0]        
__________________________________________________________________________________________________
conv2_block1_2_conv (Conv2D)    (None, 56, 56, 64)   36864       conv2_block1_2_pad[0][0]         
__________________________________________________________________________________________________
conv2_block1_2_bn (BatchNormali (None, 56, 56, 64)   256         conv2_block1_2_conv[0][0]        
__________________________________________________________________________________________________
conv2_block1_2_relu (Activation (None, 56, 56, 64)   0           conv2_block1_2_bn[0][0]          
__________________________________________________________________________________________________
conv2_block1_0_conv (Conv2D)    (None, 56, 56, 256)  16640       conv2_block1_preact_relu[0][0]   
__________________________________________________________________________________________________
conv2_block1_3_conv (Conv2D)    (None, 56, 56, 256)  16640       conv2_block1_2_relu[0][0]        
__________________________________________________________________________________________________
conv2_block1_out (Add)          (None, 56, 56, 256)  0           conv2_block1_0_conv[0][0]        
                                                                 conv2_block1_3_conv[0][0]        
__________________________________________________________________________________________________
conv2_block2_preact_bn (BatchNo (None, 56, 56, 256)  1024        conv2_block1_out[0][0]           
__________________________________________________________________________________________________
conv2_block2_preact_relu (Activ (None, 56, 56, 256)  0           conv2_block2_preact_bn[0][0]     
__________________________________________________________________________________________________
conv2_block2_1_conv (Conv2D)    (None, 56, 56, 64)   16384       conv2_block2_preact_relu[0][0]   
__________________________________________________________________________________________________
conv2_block2_1_bn (BatchNormali (None, 56, 56, 64)   256         conv2_block2_1_conv[0][0]        
__________________________________________________________________________________________________
conv2_block2_1_relu (Activation (None, 56, 56, 64)   0           conv2_block2_1_bn[0][0]          
__________________________________________________________________________________________________
conv2_block2_2_pad (ZeroPadding (None, 58, 58, 64)   0           conv2_block2_1_relu[0][0]        
__________________________________________________________________________________________________
conv2_block2_2_conv (Conv2D)    (None, 56, 56, 64)   36864       conv2_block2_2_pad[0][0]         
__________________________________________________________________________________________________
conv2_block2_2_bn (BatchNormali (None, 56, 56, 64)   256         conv2_block2_2_conv[0][0]        
__________________________________________________________________________________________________
conv2_block2_2_relu (Activation (None, 56, 56, 64)   0           conv2_block2_2_bn[0][0]          
__________________________________________________________________________________________________
conv2_block2_3_conv (Conv2D)    (None, 56, 56, 256)  16640       conv2_block2_2_relu[0][0]        
__________________________________________________________________________________________________
conv2_block2_out (Add)          (None, 56, 56, 256)  0           conv2_block1_out[0][0]           
                                                                 conv2_block2_3_conv[0][0]        
__________________________________________________________________________________________________
conv2_block3_preact_bn (BatchNo (None, 56, 56, 256)  1024        conv2_block2_out[0][0]           
__________________________________________________________________________________________________
conv2_block3_preact_relu (Activ (None, 56, 56, 256)  0           conv2_block3_preact_bn[0][0]     
__________________________________________________________________________________________________
conv2_block3_1_conv (Conv2D)    (None, 56, 56, 64)   16384       conv2_block3_preact_relu[0][0]   
__________________________________________________________________________________________________
conv2_block3_1_bn (BatchNormali (None, 56, 56, 64)   256         conv2_block3_1_conv[0][0]        
__________________________________________________________________________________________________
conv2_block3_1_relu (Activation (None, 56, 56, 64)   0           conv2_block3_1_bn[0][0]          
__________________________________________________________________________________________________
conv2_block3_2_pad (ZeroPadding (None, 58, 58, 64)   0           conv2_block3_1_relu[0][0]        
__________________________________________________________________________________________________
conv2_block3_2_conv (Conv2D)    (None, 28, 28, 64)   36864       conv2_block3_2_pad[0][0]         
__________________________________________________________________________________________________
conv2_block3_2_bn (BatchNormali (None, 28, 28, 64)   256         conv2_block3_2_conv[0][0]        
__________________________________________________________________________________________________
conv2_block3_2_relu (Activation (None, 28, 28, 64)   0           conv2_block3_2_bn[0][0]          
__________________________________________________________________________________________________
max_pooling2d_17 (MaxPooling2D) (None, 28, 28, 256)  0           conv2_block2_out[0][0]           
__________________________________________________________________________________________________
conv2_block3_3_conv (Conv2D)    (None, 28, 28, 256)  16640       conv2_block3_2_relu[0][0]        
__________________________________________________________________________________________________
conv2_block3_out (Add)          (None, 28, 28, 256)  0           max_pooling2d_17[0][0]           
                                                                 conv2_block3_3_conv[0][0]        
__________________________________________________________________________________________________
conv3_block1_preact_bn (BatchNo (None, 28, 28, 256)  1024        conv2_block3_out[0][0]           
__________________________________________________________________________________________________
conv3_block1_preact_relu (Activ (None, 28, 28, 256)  0           conv3_block1_preact_bn[0][0]     
__________________________________________________________________________________________________
conv3_block1_1_conv (Conv2D)    (None, 28, 28, 128)  32768       conv3_block1_preact_relu[0][0]   
__________________________________________________________________________________________________
conv3_block1_1_bn (BatchNormali (None, 28, 28, 128)  512         conv3_block1_1_conv[0][0]        
__________________________________________________________________________________________________
conv3_block1_1_relu (Activation (None, 28, 28, 128)  0           conv3_block1_1_bn[0][0]          
__________________________________________________________________________________________________
conv3_block1_2_pad (ZeroPadding (None, 30, 30, 128)  0           conv3_block1_1_relu[0][0]        
__________________________________________________________________________________________________
conv3_block1_2_conv (Conv2D)    (None, 28, 28, 128)  147456      conv3_block1_2_pad[0][0]         
__________________________________________________________________________________________________
conv3_block1_2_bn (BatchNormali (None, 28, 28, 128)  512         conv3_block1_2_conv[0][0]        
__________________________________________________________________________________________________
conv3_block1_2_relu (Activation (None, 28, 28, 128)  0           conv3_block1_2_bn[0][0]          
__________________________________________________________________________________________________
conv3_block1_0_conv (Conv2D)    (None, 28, 28, 512)  131584      conv3_block1_preact_relu[0][0]   
__________________________________________________________________________________________________
conv3_block1_3_conv (Conv2D)    (None, 28, 28, 512)  66048       conv3_block1_2_relu[0][0]        
__________________________________________________________________________________________________
conv3_block1_out (Add)          (None, 28, 28, 512)  0           conv3_block1_0_conv[0][0]        
                                                                 conv3_block1_3_conv[0][0]        
__________________________________________________________________________________________________
conv3_block2_preact_bn (BatchNo (None, 28, 28, 512)  2048        conv3_block1_out[0][0]           
__________________________________________________________________________________________________
conv3_block2_preact_relu (Activ (None, 28, 28, 512)  0           conv3_block2_preact_bn[0][0]     
__________________________________________________________________________________________________
conv3_block2_1_conv (Conv2D)    (None, 28, 28, 128)  65536       conv3_block2_preact_relu[0][0]   
__________________________________________________________________________________________________
conv3_block2_1_bn (BatchNormali (None, 28, 28, 128)  512         conv3_block2_1_conv[0][0]        
__________________________________________________________________________________________________
conv3_block2_1_relu (Activation (None, 28, 28, 128)  0           conv3_block2_1_bn[0][0]          
__________________________________________________________________________________________________
conv3_block2_2_pad (ZeroPadding (None, 30, 30, 128)  0           conv3_block2_1_relu[0][0]        
__________________________________________________________________________________________________
conv3_block2_2_conv (Conv2D)    (None, 28, 28, 128)  147456      conv3_block2_2_pad[0][0]         
__________________________________________________________________________________________________
conv3_block2_2_bn (BatchNormali (None, 28, 28, 128)  512         conv3_block2_2_conv[0][0]        
__________________________________________________________________________________________________
conv3_block2_2_relu (Activation (None, 28, 28, 128)  0           conv3_block2_2_bn[0][0]          
__________________________________________________________________________________________________
conv3_block2_3_conv (Conv2D)    (None, 28, 28, 512)  66048       conv3_block2_2_relu[0][0]        
__________________________________________________________________________________________________
conv3_block2_out (Add)          (None, 28, 28, 512)  0           conv3_block1_out[0][0]           
                                                                 conv3_block2_3_conv[0][0]        
__________________________________________________________________________________________________
conv3_block3_preact_bn (BatchNo (None, 28, 28, 512)  2048        conv3_block2_out[0][0]           
__________________________________________________________________________________________________
conv3_block3_preact_relu (Activ (None, 28, 28, 512)  0           conv3_block3_preact_bn[0][0]     
__________________________________________________________________________________________________
conv3_block3_1_conv (Conv2D)    (None, 28, 28, 128)  65536       conv3_block3_preact_relu[0][0]   
__________________________________________________________________________________________________
conv3_block3_1_bn (BatchNormali (None, 28, 28, 128)  512         conv3_block3_1_conv[0][0]        
__________________________________________________________________________________________________
conv3_block3_1_relu (Activation (None, 28, 28, 128)  0           conv3_block3_1_bn[0][0]          
__________________________________________________________________________________________________
conv3_block3_2_pad (ZeroPadding (None, 30, 30, 128)  0           conv3_block3_1_relu[0][0]        
__________________________________________________________________________________________________
conv3_block3_2_conv (Conv2D)    (None, 28, 28, 128)  147456      conv3_block3_2_pad[0][0]         
__________________________________________________________________________________________________
conv3_block3_2_bn (BatchNormali (None, 28, 28, 128)  512         conv3_block3_2_conv[0][0]        
__________________________________________________________________________________________________
conv3_block3_2_relu (Activation (None, 28, 28, 128)  0           conv3_block3_2_bn[0][0]          
__________________________________________________________________________________________________
conv3_block3_3_conv (Conv2D)    (None, 28, 28, 512)  66048       conv3_block3_2_relu[0][0]        
__________________________________________________________________________________________________
conv3_block3_out (Add)          (None, 28, 28, 512)  0           conv3_block2_out[0][0]           
                                                                 conv3_block3_3_conv[0][0]        
__________________________________________________________________________________________________
conv3_block4_preact_bn (BatchNo (None, 28, 28, 512)  2048        conv3_block3_out[0][0]           
__________________________________________________________________________________________________
conv3_block4_preact_relu (Activ (None, 28, 28, 512)  0           conv3_block4_preact_bn[0][0]     
__________________________________________________________________________________________________
conv3_block4_1_conv (Conv2D)    (None, 28, 28, 128)  65536       conv3_block4_preact_relu[0][0]   
__________________________________________________________________________________________________
conv3_block4_1_bn (BatchNormali (None, 28, 28, 128)  512         conv3_block4_1_conv[0][0]        
__________________________________________________________________________________________________
conv3_block4_1_relu (Activation (None, 28, 28, 128)  0           conv3_block4_1_bn[0][0]          
__________________________________________________________________________________________________
conv3_block4_2_pad (ZeroPadding (None, 30, 30, 128)  0           conv3_block4_1_relu[0][0]        
__________________________________________________________________________________________________
conv3_block4_2_conv (Conv2D)    (None, 28, 28, 128)  147456      conv3_block4_2_pad[0][0]         
__________________________________________________________________________________________________
conv3_block4_2_bn (BatchNormali (None, 28, 28, 128)  512         conv3_block4_2_conv[0][0]        
__________________________________________________________________________________________________
conv3_block4_2_relu (Activation (None, 28, 28, 128)  0           conv3_block4_2_bn[0][0]          
__________________________________________________________________________________________________
conv3_block4_3_conv (Conv2D)    (None, 28, 28, 512)  66048       conv3_block4_2_relu[0][0]        
__________________________________________________________________________________________________
conv3_block4_out (Add)          (None, 28, 28, 512)  0           conv3_block3_out[0][0]           
                                                                 conv3_block4_3_conv[0][0]        
__________________________________________________________________________________________________
conv3_block5_preact_bn (BatchNo (None, 28, 28, 512)  2048        conv3_block4_out[0][0]           
__________________________________________________________________________________________________
conv3_block5_preact_relu (Activ (None, 28, 28, 512)  0           conv3_block5_preact_bn[0][0]     
__________________________________________________________________________________________________
conv3_block5_1_conv (Conv2D)    (None, 28, 28, 128)  65536       conv3_block5_preact_relu[0][0]   
__________________________________________________________________________________________________
conv3_block5_1_bn (BatchNormali (None, 28, 28, 128)  512         conv3_block5_1_conv[0][0]        
__________________________________________________________________________________________________
conv3_block5_1_relu (Activation (None, 28, 28, 128)  0           conv3_block5_1_bn[0][0]          
__________________________________________________________________________________________________
conv3_block5_2_pad (ZeroPadding (None, 30, 30, 128)  0           conv3_block5_1_relu[0][0]        
__________________________________________________________________________________________________
conv3_block5_2_conv (Conv2D)    (None, 28, 28, 128)  147456      conv3_block5_2_pad[0][0]         
__________________________________________________________________________________________________
conv3_block5_2_bn (BatchNormali (None, 28, 28, 128)  512         conv3_block5_2_conv[0][0]        
__________________________________________________________________________________________________
conv3_block5_2_relu (Activation (None, 28, 28, 128)  0           conv3_block5_2_bn[0][0]          
__________________________________________________________________________________________________
conv3_block5_3_conv (Conv2D)    (None, 28, 28, 512)  66048       conv3_block5_2_relu[0][0]        
__________________________________________________________________________________________________
conv3_block5_out (Add)          (None, 28, 28, 512)  0           conv3_block4_out[0][0]           
                                                                 conv3_block5_3_conv[0][0]        
__________________________________________________________________________________________________
conv3_block6_preact_bn (BatchNo (None, 28, 28, 512)  2048        conv3_block5_out[0][0]           
__________________________________________________________________________________________________
conv3_block6_preact_relu (Activ (None, 28, 28, 512)  0           conv3_block6_preact_bn[0][0]     
__________________________________________________________________________________________________
conv3_block6_1_conv (Conv2D)    (None, 28, 28, 128)  65536       conv3_block6_preact_relu[0][0]   
__________________________________________________________________________________________________
conv3_block6_1_bn (BatchNormali (None, 28, 28, 128)  512         conv3_block6_1_conv[0][0]        
__________________________________________________________________________________________________
conv3_block6_1_relu (Activation (None, 28, 28, 128)  0           conv3_block6_1_bn[0][0]          
__________________________________________________________________________________________________
conv3_block6_2_pad (ZeroPadding (None, 30, 30, 128)  0           conv3_block6_1_relu[0][0]        
__________________________________________________________________________________________________
conv3_block6_2_conv (Conv2D)    (None, 28, 28, 128)  147456      conv3_block6_2_pad[0][0]         
__________________________________________________________________________________________________
conv3_block6_2_bn (BatchNormali (None, 28, 28, 128)  512         conv3_block6_2_conv[0][0]        
__________________________________________________________________________________________________
conv3_block6_2_relu (Activation (None, 28, 28, 128)  0           conv3_block6_2_bn[0][0]          
__________________________________________________________________________________________________
conv3_block6_3_conv (Conv2D)    (None, 28, 28, 512)  66048       conv3_block6_2_relu[0][0]        
__________________________________________________________________________________________________
conv3_block6_out (Add)          (None, 28, 28, 512)  0           conv3_block5_out[0][0]           
                                                                 conv3_block6_3_conv[0][0]        
__________________________________________________________________________________________________
conv3_block7_preact_bn (BatchNo (None, 28, 28, 512)  2048        conv3_block6_out[0][0]           
__________________________________________________________________________________________________
conv3_block7_preact_relu (Activ (None, 28, 28, 512)  0           conv3_block7_preact_bn[0][0]     
__________________________________________________________________________________________________
conv3_block7_1_conv (Conv2D)    (None, 28, 28, 128)  65536       conv3_block7_preact_relu[0][0]   
__________________________________________________________________________________________________
conv3_block7_1_bn (BatchNormali (None, 28, 28, 128)  512         conv3_block7_1_conv[0][0]        
__________________________________________________________________________________________________
conv3_block7_1_relu (Activation (None, 28, 28, 128)  0           conv3_block7_1_bn[0][0]          
__________________________________________________________________________________________________
conv3_block7_2_pad (ZeroPadding (None, 30, 30, 128)  0           conv3_block7_1_relu[0][0]        
__________________________________________________________________________________________________
conv3_block7_2_conv (Conv2D)    (None, 28, 28, 128)  147456      conv3_block7_2_pad[0][0]         
__________________________________________________________________________________________________
conv3_block7_2_bn (BatchNormali (None, 28, 28, 128)  512         conv3_block7_2_conv[0][0]        
__________________________________________________________________________________________________
conv3_block7_2_relu (Activation (None, 28, 28, 128)  0           conv3_block7_2_bn[0][0]          
__________________________________________________________________________________________________
conv3_block7_3_conv (Conv2D)    (None, 28, 28, 512)  66048       conv3_block7_2_relu[0][0]        
__________________________________________________________________________________________________
conv3_block7_out (Add)          (None, 28, 28, 512)  0           conv3_block6_out[0][0]           
                                                                 conv3_block7_3_conv[0][0]        
__________________________________________________________________________________________________
conv3_block8_preact_bn (BatchNo (None, 28, 28, 512)  2048        conv3_block7_out[0][0]           
__________________________________________________________________________________________________
conv3_block8_preact_relu (Activ (None, 28, 28, 512)  0           conv3_block8_preact_bn[0][0]     
__________________________________________________________________________________________________
conv3_block8_1_conv (Conv2D)    (None, 28, 28, 128)  65536       conv3_block8_preact_relu[0][0]   
__________________________________________________________________________________________________
conv3_block8_1_bn (BatchNormali (None, 28, 28, 128)  512         conv3_block8_1_conv[0][0]        
__________________________________________________________________________________________________
conv3_block8_1_relu (Activation (None, 28, 28, 128)  0           conv3_block8_1_bn[0][0]          
__________________________________________________________________________________________________
conv3_block8_2_pad (ZeroPadding (None, 30, 30, 128)  0           conv3_block8_1_relu[0][0]        
__________________________________________________________________________________________________
conv3_block8_2_conv (Conv2D)    (None, 14, 14, 128)  147456      conv3_block8_2_pad[0][0]         
__________________________________________________________________________________________________
conv3_block8_2_bn (BatchNormali (None, 14, 14, 128)  512         conv3_block8_2_conv[0][0]        
__________________________________________________________________________________________________
conv3_block8_2_relu (Activation (None, 14, 14, 128)  0           conv3_block8_2_bn[0][0]          
__________________________________________________________________________________________________
max_pooling2d_18 (MaxPooling2D) (None, 14, 14, 512)  0           conv3_block7_out[0][0]           
__________________________________________________________________________________________________
conv3_block8_3_conv (Conv2D)    (None, 14, 14, 512)  66048       conv3_block8_2_relu[0][0]        
__________________________________________________________________________________________________
conv3_block8_out (Add)          (None, 14, 14, 512)  0           max_pooling2d_18[0][0]           
                                                                 conv3_block8_3_conv[0][0]        
__________________________________________________________________________________________________
conv4_block1_preact_bn (BatchNo (None, 14, 14, 512)  2048        conv3_block8_out[0][0]           
__________________________________________________________________________________________________
conv4_block1_preact_relu (Activ (None, 14, 14, 512)  0           conv4_block1_preact_bn[0][0]     
__________________________________________________________________________________________________
conv4_block1_1_conv (Conv2D)    (None, 14, 14, 256)  131072      conv4_block1_preact_relu[0][0]   
__________________________________________________________________________________________________
conv4_block1_1_bn (BatchNormali (None, 14, 14, 256)  1024        conv4_block1_1_conv[0][0]        
__________________________________________________________________________________________________
conv4_block1_1_relu (Activation (None, 14, 14, 256)  0           conv4_block1_1_bn[0][0]          
__________________________________________________________________________________________________
conv4_block1_2_pad (ZeroPadding (None, 16, 16, 256)  0           conv4_block1_1_relu[0][0]        
__________________________________________________________________________________________________
conv4_block1_2_conv (Conv2D)    (None, 14, 14, 256)  589824      conv4_block1_2_pad[0][0]         
__________________________________________________________________________________________________
conv4_block1_2_bn (BatchNormali (None, 14, 14, 256)  1024        conv4_block1_2_conv[0][0]        
__________________________________________________________________________________________________
conv4_block1_2_relu (Activation (None, 14, 14, 256)  0           conv4_block1_2_bn[0][0]          
__________________________________________________________________________________________________
conv4_block1_0_conv (Conv2D)    (None, 14, 14, 1024) 525312      conv4_block1_preact_relu[0][0]   
__________________________________________________________________________________________________
conv4_block1_3_conv (Conv2D)    (None, 14, 14, 1024) 263168      conv4_block1_2_relu[0][0]        
__________________________________________________________________________________________________
conv4_block1_out (Add)          (None, 14, 14, 1024) 0           conv4_block1_0_conv[0][0]        
                                                                 conv4_block1_3_conv[0][0]        
__________________________________________________________________________________________________
conv4_block2_preact_bn (BatchNo (None, 14, 14, 1024) 4096        conv4_block1_out[0][0]           
__________________________________________________________________________________________________
conv4_block2_preact_relu (Activ (None, 14, 14, 1024) 0           conv4_block2_preact_bn[0][0]     
__________________________________________________________________________________________________
conv4_block2_1_conv (Conv2D)    (None, 14, 14, 256)  262144      conv4_block2_preact_relu[0][0]   
__________________________________________________________________________________________________
conv4_block2_1_bn (BatchNormali (None, 14, 14, 256)  1024        conv4_block2_1_conv[0][0]        
__________________________________________________________________________________________________
conv4_block2_1_relu (Activation (None, 14, 14, 256)  0           conv4_block2_1_bn[0][0]          
__________________________________________________________________________________________________
conv4_block2_2_pad (ZeroPadding (None, 16, 16, 256)  0           conv4_block2_1_relu[0][0]        
__________________________________________________________________________________________________
conv4_block2_2_conv (Conv2D)    (None, 14, 14, 256)  589824      conv4_block2_2_pad[0][0]         
__________________________________________________________________________________________________
conv4_block2_2_bn (BatchNormali (None, 14, 14, 256)  1024        conv4_block2_2_conv[0][0]        
__________________________________________________________________________________________________
conv4_block2_2_relu (Activation (None, 14, 14, 256)  0           conv4_block2_2_bn[0][0]          
__________________________________________________________________________________________________
conv4_block2_3_conv (Conv2D)    (None, 14, 14, 1024) 263168      conv4_block2_2_relu[0][0]        
__________________________________________________________________________________________________
conv4_block2_out (Add)          (None, 14, 14, 1024) 0           conv4_block1_out[0][0]           
                                                                 conv4_block2_3_conv[0][0]        
__________________________________________________________________________________________________
conv4_block3_preact_bn (BatchNo (None, 14, 14, 1024) 4096        conv4_block2_out[0][0]           
__________________________________________________________________________________________________
conv4_block3_preact_relu (Activ (None, 14, 14, 1024) 0           conv4_block3_preact_bn[0][0]     
__________________________________________________________________________________________________
conv4_block3_1_conv (Conv2D)    (None, 14, 14, 256)  262144      conv4_block3_preact_relu[0][0]   
__________________________________________________________________________________________________
conv4_block3_1_bn (BatchNormali (None, 14, 14, 256)  1024        conv4_block3_1_conv[0][0]        
__________________________________________________________________________________________________
conv4_block3_1_relu (Activation (None, 14, 14, 256)  0           conv4_block3_1_bn[0][0]          
__________________________________________________________________________________________________
conv4_block3_2_pad (ZeroPadding (None, 16, 16, 256)  0           conv4_block3_1_relu[0][0]        
__________________________________________________________________________________________________
conv4_block3_2_conv (Conv2D)    (None, 14, 14, 256)  589824      conv4_block3_2_pad[0][0]         
__________________________________________________________________________________________________
conv4_block3_2_bn (BatchNormali (None, 14, 14, 256)  1024        conv4_block3_2_conv[0][0]        
__________________________________________________________________________________________________
conv4_block3_2_relu (Activation (None, 14, 14, 256)  0           conv4_block3_2_bn[0][0]          
__________________________________________________________________________________________________
conv4_block3_3_conv (Conv2D)    (None, 14, 14, 1024) 263168      conv4_block3_2_relu[0][0]        
__________________________________________________________________________________________________
conv4_block3_out (Add)          (None, 14, 14, 1024) 0           conv4_block2_out[0][0]           
                                                                 conv4_block3_3_conv[0][0]        
__________________________________________________________________________________________________
conv4_block4_preact_bn (BatchNo (None, 14, 14, 1024) 4096        conv4_block3_out[0][0]           
__________________________________________________________________________________________________
conv4_block4_preact_relu (Activ (None, 14, 14, 1024) 0           conv4_block4_preact_bn[0][0]     
__________________________________________________________________________________________________
conv4_block4_1_conv (Conv2D)    (None, 14, 14, 256)  262144      conv4_block4_preact_relu[0][0]   
__________________________________________________________________________________________________
conv4_block4_1_bn (BatchNormali (None, 14, 14, 256)  1024        conv4_block4_1_conv[0][0]        
__________________________________________________________________________________________________
conv4_block4_1_relu (Activation (None, 14, 14, 256)  0           conv4_block4_1_bn[0][0]          
__________________________________________________________________________________________________
conv4_block4_2_pad (ZeroPadding (None, 16, 16, 256)  0           conv4_block4_1_relu[0][0]        
__________________________________________________________________________________________________
conv4_block4_2_conv (Conv2D)    (None, 14, 14, 256)  589824      conv4_block4_2_pad[0][0]         
__________________________________________________________________________________________________
conv4_block4_2_bn (BatchNormali (None, 14, 14, 256)  1024        conv4_block4_2_conv[0][0]        
__________________________________________________________________________________________________
conv4_block4_2_relu (Activation (None, 14, 14, 256)  0           conv4_block4_2_bn[0][0]          
__________________________________________________________________________________________________
conv4_block4_3_conv (Conv2D)    (None, 14, 14, 1024) 263168      conv4_block4_2_relu[0][0]        
__________________________________________________________________________________________________
conv4_block4_out (Add)          (None, 14, 14, 1024) 0           conv4_block3_out[0][0]           
                                                                 conv4_block4_3_conv[0][0]        
__________________________________________________________________________________________________
conv4_block5_preact_bn (BatchNo (None, 14, 14, 1024) 4096        conv4_block4_out[0][0]           
__________________________________________________________________________________________________
conv4_block5_preact_relu (Activ (None, 14, 14, 1024) 0           conv4_block5_preact_bn[0][0]     
__________________________________________________________________________________________________
conv4_block5_1_conv (Conv2D)    (None, 14, 14, 256)  262144      conv4_block5_preact_relu[0][0]   
__________________________________________________________________________________________________
conv4_block5_1_bn (BatchNormali (None, 14, 14, 256)  1024        conv4_block5_1_conv[0][0]        
__________________________________________________________________________________________________
conv4_block5_1_relu (Activation (None, 14, 14, 256)  0           conv4_block5_1_bn[0][0]          
__________________________________________________________________________________________________
conv4_block5_2_pad (ZeroPadding (None, 16, 16, 256)  0           conv4_block5_1_relu[0][0]        
__________________________________________________________________________________________________
conv4_block5_2_conv (Conv2D)    (None, 14, 14, 256)  589824      conv4_block5_2_pad[0][0]         
__________________________________________________________________________________________________
conv4_block5_2_bn (BatchNormali (None, 14, 14, 256)  1024        conv4_block5_2_conv[0][0]        
__________________________________________________________________________________________________
conv4_block5_2_relu (Activation (None, 14, 14, 256)  0           conv4_block5_2_bn[0][0]          
__________________________________________________________________________________________________
conv4_block5_3_conv (Conv2D)    (None, 14, 14, 1024) 263168      conv4_block5_2_relu[0][0]        
__________________________________________________________________________________________________
conv4_block5_out (Add)          (None, 14, 14, 1024) 0           conv4_block4_out[0][0]           
                                                                 conv4_block5_3_conv[0][0]        
__________________________________________________________________________________________________
conv4_block6_preact_bn (BatchNo (None, 14, 14, 1024) 4096        conv4_block5_out[0][0]           
__________________________________________________________________________________________________
conv4_block6_preact_relu (Activ (None, 14, 14, 1024) 0           conv4_block6_preact_bn[0][0]     
__________________________________________________________________________________________________
conv4_block6_1_conv (Conv2D)    (None, 14, 14, 256)  262144      conv4_block6_preact_relu[0][0]   
__________________________________________________________________________________________________
conv4_block6_1_bn (BatchNormali (None, 14, 14, 256)  1024        conv4_block6_1_conv[0][0]        
__________________________________________________________________________________________________
conv4_block6_1_relu (Activation (None, 14, 14, 256)  0           conv4_block6_1_bn[0][0]          
__________________________________________________________________________________________________
conv4_block6_2_pad (ZeroPadding (None, 16, 16, 256)  0           conv4_block6_1_relu[0][0]        
__________________________________________________________________________________________________
conv4_block6_2_conv (Conv2D)    (None, 14, 14, 256)  589824      conv4_block6_2_pad[0][0]         
__________________________________________________________________________________________________
conv4_block6_2_bn (BatchNormali (None, 14, 14, 256)  1024        conv4_block6_2_conv[0][0]        
__________________________________________________________________________________________________
conv4_block6_2_relu (Activation (None, 14, 14, 256)  0           conv4_block6_2_bn[0][0]          
__________________________________________________________________________________________________
conv4_block6_3_conv (Conv2D)    (None, 14, 14, 1024) 263168      conv4_block6_2_relu[0][0]        
__________________________________________________________________________________________________
conv4_block6_out (Add)          (None, 14, 14, 1024) 0           conv4_block5_out[0][0]           
                                                                 conv4_block6_3_conv[0][0]        
__________________________________________________________________________________________________
conv4_block7_preact_bn (BatchNo (None, 14, 14, 1024) 4096        conv4_block6_out[0][0]           
__________________________________________________________________________________________________
conv4_block7_preact_relu (Activ (None, 14, 14, 1024) 0           conv4_block7_preact_bn[0][0]     
__________________________________________________________________________________________________
conv4_block7_1_conv (Conv2D)    (None, 14, 14, 256)  262144      conv4_block7_preact_relu[0][0]   
__________________________________________________________________________________________________
conv4_block7_1_bn (BatchNormali (None, 14, 14, 256)  1024        conv4_block7_1_conv[0][0]        
__________________________________________________________________________________________________
conv4_block7_1_relu (Activation (None, 14, 14, 256)  0           conv4_block7_1_bn[0][0]          
__________________________________________________________________________________________________
conv4_block7_2_pad (ZeroPadding (None, 16, 16, 256)  0           conv4_block7_1_relu[0][0]        
__________________________________________________________________________________________________
conv4_block7_2_conv (Conv2D)    (None, 14, 14, 256)  589824      conv4_block7_2_pad[0][0]         
__________________________________________________________________________________________________
conv4_block7_2_bn (BatchNormali (None, 14, 14, 256)  1024        conv4_block7_2_conv[0][0]        
__________________________________________________________________________________________________
conv4_block7_2_relu (Activation (None, 14, 14, 256)  0           conv4_block7_2_bn[0][0]          
__________________________________________________________________________________________________
conv4_block7_3_conv (Conv2D)    (None, 14, 14, 1024) 263168      conv4_block7_2_relu[0][0]        
__________________________________________________________________________________________________
conv4_block7_out (Add)          (None, 14, 14, 1024) 0           conv4_block6_out[0][0]           
                                                                 conv4_block7_3_conv[0][0]        
__________________________________________________________________________________________________
conv4_block8_preact_bn (BatchNo (None, 14, 14, 1024) 4096        conv4_block7_out[0][0]           
__________________________________________________________________________________________________
conv4_block8_preact_relu (Activ (None, 14, 14, 1024) 0           conv4_block8_preact_bn[0][0]     
__________________________________________________________________________________________________
conv4_block8_1_conv (Conv2D)    (None, 14, 14, 256)  262144      conv4_block8_preact_relu[0][0]   
__________________________________________________________________________________________________
conv4_block8_1_bn (BatchNormali (None, 14, 14, 256)  1024        conv4_block8_1_conv[0][0]        
__________________________________________________________________________________________________
conv4_block8_1_relu (Activation (None, 14, 14, 256)  0           conv4_block8_1_bn[0][0]          
__________________________________________________________________________________________________
conv4_block8_2_pad (ZeroPadding (None, 16, 16, 256)  0           conv4_block8_1_relu[0][0]        
__________________________________________________________________________________________________
conv4_block8_2_conv (Conv2D)    (None, 14, 14, 256)  589824      conv4_block8_2_pad[0][0]         
__________________________________________________________________________________________________
conv4_block8_2_bn (BatchNormali (None, 14, 14, 256)  1024        conv4_block8_2_conv[0][0]        
__________________________________________________________________________________________________
conv4_block8_2_relu (Activation (None, 14, 14, 256)  0           conv4_block8_2_bn[0][0]          
__________________________________________________________________________________________________
conv4_block8_3_conv (Conv2D)    (None, 14, 14, 1024) 263168      conv4_block8_2_relu[0][0]        
__________________________________________________________________________________________________
conv4_block8_out (Add)          (None, 14, 14, 1024) 0           conv4_block7_out[0][0]           
                                                                 conv4_block8_3_conv[0][0]        
__________________________________________________________________________________________________
conv4_block9_preact_bn (BatchNo (None, 14, 14, 1024) 4096        conv4_block8_out[0][0]           
__________________________________________________________________________________________________
conv4_block9_preact_relu (Activ (None, 14, 14, 1024) 0           conv4_block9_preact_bn[0][0]     
__________________________________________________________________________________________________
conv4_block9_1_conv (Conv2D)    (None, 14, 14, 256)  262144      conv4_block9_preact_relu[0][0]   
__________________________________________________________________________________________________
conv4_block9_1_bn (BatchNormali (None, 14, 14, 256)  1024        conv4_block9_1_conv[0][0]        
__________________________________________________________________________________________________
conv4_block9_1_relu (Activation (None, 14, 14, 256)  0           conv4_block9_1_bn[0][0]          
__________________________________________________________________________________________________
conv4_block9_2_pad (ZeroPadding (None, 16, 16, 256)  0           conv4_block9_1_relu[0][0]        
__________________________________________________________________________________________________
conv4_block9_2_conv (Conv2D)    (None, 14, 14, 256)  589824      conv4_block9_2_pad[0][0]         
__________________________________________________________________________________________________
conv4_block9_2_bn (BatchNormali (None, 14, 14, 256)  1024        conv4_block9_2_conv[0][0]        
__________________________________________________________________________________________________
conv4_block9_2_relu (Activation (None, 14, 14, 256)  0           conv4_block9_2_bn[0][0]          
__________________________________________________________________________________________________
conv4_block9_3_conv (Conv2D)    (None, 14, 14, 1024) 263168      conv4_block9_2_relu[0][0]        
__________________________________________________________________________________________________
conv4_block9_out (Add)          (None, 14, 14, 1024) 0           conv4_block8_out[0][0]           
                                                                 conv4_block9_3_conv[0][0]        
__________________________________________________________________________________________________
conv4_block10_preact_bn (BatchN (None, 14, 14, 1024) 4096        conv4_block9_out[0][0]           
__________________________________________________________________________________________________
conv4_block10_preact_relu (Acti (None, 14, 14, 1024) 0           conv4_block10_preact_bn[0][0]    
__________________________________________________________________________________________________
conv4_block10_1_conv (Conv2D)   (None, 14, 14, 256)  262144      conv4_block10_preact_relu[0][0]  
__________________________________________________________________________________________________
conv4_block10_1_bn (BatchNormal (None, 14, 14, 256)  1024        conv4_block10_1_conv[0][0]       
__________________________________________________________________________________________________
conv4_block10_1_relu (Activatio (None, 14, 14, 256)  0           conv4_block10_1_bn[0][0]         
__________________________________________________________________________________________________
conv4_block10_2_pad (ZeroPaddin (None, 16, 16, 256)  0           conv4_block10_1_relu[0][0]       
__________________________________________________________________________________________________
conv4_block10_2_conv (Conv2D)   (None, 14, 14, 256)  589824      conv4_block10_2_pad[0][0]        
__________________________________________________________________________________________________
conv4_block10_2_bn (BatchNormal (None, 14, 14, 256)  1024        conv4_block10_2_conv[0][0]       
__________________________________________________________________________________________________
conv4_block10_2_relu (Activatio (None, 14, 14, 256)  0           conv4_block10_2_bn[0][0]         
__________________________________________________________________________________________________
conv4_block10_3_conv (Conv2D)   (None, 14, 14, 1024) 263168      conv4_block10_2_relu[0][0]       
__________________________________________________________________________________________________
conv4_block10_out (Add)         (None, 14, 14, 1024) 0           conv4_block9_out[0][0]           
                                                                 conv4_block10_3_conv[0][0]       
__________________________________________________________________________________________________
conv4_block11_preact_bn (BatchN (None, 14, 14, 1024) 4096        conv4_block10_out[0][0]          
__________________________________________________________________________________________________
conv4_block11_preact_relu (Acti (None, 14, 14, 1024) 0           conv4_block11_preact_bn[0][0]    
__________________________________________________________________________________________________
conv4_block11_1_conv (Conv2D)   (None, 14, 14, 256)  262144      conv4_block11_preact_relu[0][0]  
__________________________________________________________________________________________________
conv4_block11_1_bn (BatchNormal (None, 14, 14, 256)  1024        conv4_block11_1_conv[0][0]       
__________________________________________________________________________________________________
conv4_block11_1_relu (Activatio (None, 14, 14, 256)  0           conv4_block11_1_bn[0][0]         
__________________________________________________________________________________________________
conv4_block11_2_pad (ZeroPaddin (None, 16, 16, 256)  0           conv4_block11_1_relu[0][0]       
__________________________________________________________________________________________________
conv4_block11_2_conv (Conv2D)   (None, 14, 14, 256)  589824      conv4_block11_2_pad[0][0]        
__________________________________________________________________________________________________
conv4_block11_2_bn (BatchNormal (None, 14, 14, 256)  1024        conv4_block11_2_conv[0][0]       
__________________________________________________________________________________________________
conv4_block11_2_relu (Activatio (None, 14, 14, 256)  0           conv4_block11_2_bn[0][0]         
__________________________________________________________________________________________________
conv4_block11_3_conv (Conv2D)   (None, 14, 14, 1024) 263168      conv4_block11_2_relu[0][0]       
__________________________________________________________________________________________________
conv4_block11_out (Add)         (None, 14, 14, 1024) 0           conv4_block10_out[0][0]          
                                                                 conv4_block11_3_conv[0][0]       
__________________________________________________________________________________________________
conv4_block12_preact_bn (BatchN (None, 14, 14, 1024) 4096        conv4_block11_out[0][0]          
__________________________________________________________________________________________________
conv4_block12_preact_relu (Acti (None, 14, 14, 1024) 0           conv4_block12_preact_bn[0][0]    
__________________________________________________________________________________________________
conv4_block12_1_conv (Conv2D)   (None, 14, 14, 256)  262144      conv4_block12_preact_relu[0][0]  
__________________________________________________________________________________________________
conv4_block12_1_bn (BatchNormal (None, 14, 14, 256)  1024        conv4_block12_1_conv[0][0]       
__________________________________________________________________________________________________
conv4_block12_1_relu (Activatio (None, 14, 14, 256)  0           conv4_block12_1_bn[0][0]         
__________________________________________________________________________________________________
conv4_block12_2_pad (ZeroPaddin (None, 16, 16, 256)  0           conv4_block12_1_relu[0][0]       
__________________________________________________________________________________________________
conv4_block12_2_conv (Conv2D)   (None, 14, 14, 256)  589824      conv4_block12_2_pad[0][0]        
__________________________________________________________________________________________________
conv4_block12_2_bn (BatchNormal (None, 14, 14, 256)  1024        conv4_block12_2_conv[0][0]       
__________________________________________________________________________________________________
conv4_block12_2_relu (Activatio (None, 14, 14, 256)  0           conv4_block12_2_bn[0][0]         
__________________________________________________________________________________________________
conv4_block12_3_conv (Conv2D)   (None, 14, 14, 1024) 263168      conv4_block12_2_relu[0][0]       
__________________________________________________________________________________________________
conv4_block12_out (Add)         (None, 14, 14, 1024) 0           conv4_block11_out[0][0]          
                                                                 conv4_block12_3_conv[0][0]       
__________________________________________________________________________________________________
conv4_block13_preact_bn (BatchN (None, 14, 14, 1024) 4096        conv4_block12_out[0][0]          
__________________________________________________________________________________________________
conv4_block13_preact_relu (Acti (None, 14, 14, 1024) 0           conv4_block13_preact_bn[0][0]    
__________________________________________________________________________________________________
conv4_block13_1_conv (Conv2D)   (None, 14, 14, 256)  262144      conv4_block13_preact_relu[0][0]  
__________________________________________________________________________________________________
conv4_block13_1_bn (BatchNormal (None, 14, 14, 256)  1024        conv4_block13_1_conv[0][0]       
__________________________________________________________________________________________________
conv4_block13_1_relu (Activatio (None, 14, 14, 256)  0           conv4_block13_1_bn[0][0]         
__________________________________________________________________________________________________
conv4_block13_2_pad (ZeroPaddin (None, 16, 16, 256)  0           conv4_block13_1_relu[0][0]       
__________________________________________________________________________________________________
conv4_block13_2_conv (Conv2D)   (None, 14, 14, 256)  589824      conv4_block13_2_pad[0][0]        
__________________________________________________________________________________________________
conv4_block13_2_bn (BatchNormal (None, 14, 14, 256)  1024        conv4_block13_2_conv[0][0]       
__________________________________________________________________________________________________
conv4_block13_2_relu (Activatio (None, 14, 14, 256)  0           conv4_block13_2_bn[0][0]         
__________________________________________________________________________________________________
conv4_block13_3_conv (Conv2D)   (None, 14, 14, 1024) 263168      conv4_block13_2_relu[0][0]       
__________________________________________________________________________________________________
conv4_block13_out (Add)         (None, 14, 14, 1024) 0           conv4_block12_out[0][0]          
                                                                 conv4_block13_3_conv[0][0]       
__________________________________________________________________________________________________
conv4_block14_preact_bn (BatchN (None, 14, 14, 1024) 4096        conv4_block13_out[0][0]          
__________________________________________________________________________________________________
conv4_block14_preact_relu (Acti (None, 14, 14, 1024) 0           conv4_block14_preact_bn[0][0]    
__________________________________________________________________________________________________
conv4_block14_1_conv (Conv2D)   (None, 14, 14, 256)  262144      conv4_block14_preact_relu[0][0]  
__________________________________________________________________________________________________
conv4_block14_1_bn (BatchNormal (None, 14, 14, 256)  1024        conv4_block14_1_conv[0][0]       
__________________________________________________________________________________________________
conv4_block14_1_relu (Activatio (None, 14, 14, 256)  0           conv4_block14_1_bn[0][0]         
__________________________________________________________________________________________________
conv4_block14_2_pad (ZeroPaddin (None, 16, 16, 256)  0           conv4_block14_1_relu[0][0]       
__________________________________________________________________________________________________
conv4_block14_2_conv (Conv2D)   (None, 14, 14, 256)  589824      conv4_block14_2_pad[0][0]        
__________________________________________________________________________________________________
conv4_block14_2_bn (BatchNormal (None, 14, 14, 256)  1024        conv4_block14_2_conv[0][0]       
__________________________________________________________________________________________________
conv4_block14_2_relu (Activatio (None, 14, 14, 256)  0           conv4_block14_2_bn[0][0]         
__________________________________________________________________________________________________
conv4_block14_3_conv (Conv2D)   (None, 14, 14, 1024) 263168      conv4_block14_2_relu[0][0]       
__________________________________________________________________________________________________
conv4_block14_out (Add)         (None, 14, 14, 1024) 0           conv4_block13_out[0][0]          
                                                                 conv4_block14_3_conv[0][0]       
__________________________________________________________________________________________________
conv4_block15_preact_bn (BatchN (None, 14, 14, 1024) 4096        conv4_block14_out[0][0]          
__________________________________________________________________________________________________
conv4_block15_preact_relu (Acti (None, 14, 14, 1024) 0           conv4_block15_preact_bn[0][0]    
__________________________________________________________________________________________________
conv4_block15_1_conv (Conv2D)   (None, 14, 14, 256)  262144      conv4_block15_preact_relu[0][0]  
__________________________________________________________________________________________________
conv4_block15_1_bn (BatchNormal (None, 14, 14, 256)  1024        conv4_block15_1_conv[0][0]       
__________________________________________________________________________________________________
conv4_block15_1_relu (Activatio (None, 14, 14, 256)  0           conv4_block15_1_bn[0][0]         
__________________________________________________________________________________________________
conv4_block15_2_pad (ZeroPaddin (None, 16, 16, 256)  0           conv4_block15_1_relu[0][0]       
__________________________________________________________________________________________________
conv4_block15_2_conv (Conv2D)   (None, 14, 14, 256)  589824      conv4_block15_2_pad[0][0]        
__________________________________________________________________________________________________
conv4_block15_2_bn (BatchNormal (None, 14, 14, 256)  1024        conv4_block15_2_conv[0][0]       
__________________________________________________________________________________________________
conv4_block15_2_relu (Activatio (None, 14, 14, 256)  0           conv4_block15_2_bn[0][0]         
__________________________________________________________________________________________________
conv4_block15_3_conv (Conv2D)   (None, 14, 14, 1024) 263168      conv4_block15_2_relu[0][0]       
__________________________________________________________________________________________________
conv4_block15_out (Add)         (None, 14, 14, 1024) 0           conv4_block14_out[0][0]          
                                                                 conv4_block15_3_conv[0][0]       
__________________________________________________________________________________________________
conv4_block16_preact_bn (BatchN (None, 14, 14, 1024) 4096        conv4_block15_out[0][0]          
__________________________________________________________________________________________________
conv4_block16_preact_relu (Acti (None, 14, 14, 1024) 0           conv4_block16_preact_bn[0][0]    
__________________________________________________________________________________________________
conv4_block16_1_conv (Conv2D)   (None, 14, 14, 256)  262144      conv4_block16_preact_relu[0][0]  
__________________________________________________________________________________________________
conv4_block16_1_bn (BatchNormal (None, 14, 14, 256)  1024        conv4_block16_1_conv[0][0]       
__________________________________________________________________________________________________
conv4_block16_1_relu (Activatio (None, 14, 14, 256)  0           conv4_block16_1_bn[0][0]         
__________________________________________________________________________________________________
conv4_block16_2_pad (ZeroPaddin (None, 16, 16, 256)  0           conv4_block16_1_relu[0][0]       
__________________________________________________________________________________________________
conv4_block16_2_conv (Conv2D)   (None, 14, 14, 256)  589824      conv4_block16_2_pad[0][0]        
__________________________________________________________________________________________________
conv4_block16_2_bn (BatchNormal (None, 14, 14, 256)  1024        conv4_block16_2_conv[0][0]       
__________________________________________________________________________________________________
conv4_block16_2_relu (Activatio (None, 14, 14, 256)  0           conv4_block16_2_bn[0][0]         
__________________________________________________________________________________________________
conv4_block16_3_conv (Conv2D)   (None, 14, 14, 1024) 263168      conv4_block16_2_relu[0][0]       
__________________________________________________________________________________________________
conv4_block16_out (Add)         (None, 14, 14, 1024) 0           conv4_block15_out[0][0]          
                                                                 conv4_block16_3_conv[0][0]       
__________________________________________________________________________________________________
conv4_block17_preact_bn (BatchN (None, 14, 14, 1024) 4096        conv4_block16_out[0][0]          
__________________________________________________________________________________________________
conv4_block17_preact_relu (Acti (None, 14, 14, 1024) 0           conv4_block17_preact_bn[0][0]    
__________________________________________________________________________________________________
conv4_block17_1_conv (Conv2D)   (None, 14, 14, 256)  262144      conv4_block17_preact_relu[0][0]  
__________________________________________________________________________________________________
conv4_block17_1_bn (BatchNormal (None, 14, 14, 256)  1024        conv4_block17_1_conv[0][0]       
__________________________________________________________________________________________________
conv4_block17_1_relu (Activatio (None, 14, 14, 256)  0           conv4_block17_1_bn[0][0]         
__________________________________________________________________________________________________
conv4_block17_2_pad (ZeroPaddin (None, 16, 16, 256)  0           conv4_block17_1_relu[0][0]       
__________________________________________________________________________________________________
conv4_block17_2_conv (Conv2D)   (None, 14, 14, 256)  589824      conv4_block17_2_pad[0][0]        
__________________________________________________________________________________________________
conv4_block17_2_bn (BatchNormal (None, 14, 14, 256)  1024        conv4_block17_2_conv[0][0]       
__________________________________________________________________________________________________
conv4_block17_2_relu (Activatio (None, 14, 14, 256)  0           conv4_block17_2_bn[0][0]         
__________________________________________________________________________________________________
conv4_block17_3_conv (Conv2D)   (None, 14, 14, 1024) 263168      conv4_block17_2_relu[0][0]       
__________________________________________________________________________________________________
conv4_block17_out (Add)         (None, 14, 14, 1024) 0           conv4_block16_out[0][0]          
                                                                 conv4_block17_3_conv[0][0]       
__________________________________________________________________________________________________
conv4_block18_preact_bn (BatchN (None, 14, 14, 1024) 4096        conv4_block17_out[0][0]          
__________________________________________________________________________________________________
conv4_block18_preact_relu (Acti (None, 14, 14, 1024) 0           conv4_block18_preact_bn[0][0]    
__________________________________________________________________________________________________
conv4_block18_1_conv (Conv2D)   (None, 14, 14, 256)  262144      conv4_block18_preact_relu[0][0]  
__________________________________________________________________________________________________
conv4_block18_1_bn (BatchNormal (None, 14, 14, 256)  1024        conv4_block18_1_conv[0][0]       
__________________________________________________________________________________________________
conv4_block18_1_relu (Activatio (None, 14, 14, 256)  0           conv4_block18_1_bn[0][0]         
__________________________________________________________________________________________________
conv4_block18_2_pad (ZeroPaddin (None, 16, 16, 256)  0           conv4_block18_1_relu[0][0]       
__________________________________________________________________________________________________
conv4_block18_2_conv (Conv2D)   (None, 14, 14, 256)  589824      conv4_block18_2_pad[0][0]        
__________________________________________________________________________________________________
conv4_block18_2_bn (BatchNormal (None, 14, 14, 256)  1024        conv4_block18_2_conv[0][0]       
__________________________________________________________________________________________________
conv4_block18_2_relu (Activatio (None, 14, 14, 256)  0           conv4_block18_2_bn[0][0]         
__________________________________________________________________________________________________
conv4_block18_3_conv (Conv2D)   (None, 14, 14, 1024) 263168      conv4_block18_2_relu[0][0]       
__________________________________________________________________________________________________
conv4_block18_out (Add)         (None, 14, 14, 1024) 0           conv4_block17_out[0][0]          
                                                                 conv4_block18_3_conv[0][0]       
__________________________________________________________________________________________________
conv4_block19_preact_bn (BatchN (None, 14, 14, 1024) 4096        conv4_block18_out[0][0]          
__________________________________________________________________________________________________
conv4_block19_preact_relu (Acti (None, 14, 14, 1024) 0           conv4_block19_preact_bn[0][0]    
__________________________________________________________________________________________________
conv4_block19_1_conv (Conv2D)   (None, 14, 14, 256)  262144      conv4_block19_preact_relu[0][0]  
__________________________________________________________________________________________________
conv4_block19_1_bn (BatchNormal (None, 14, 14, 256)  1024        conv4_block19_1_conv[0][0]       
__________________________________________________________________________________________________
conv4_block19_1_relu (Activatio (None, 14, 14, 256)  0           conv4_block19_1_bn[0][0]         
__________________________________________________________________________________________________
conv4_block19_2_pad (ZeroPaddin (None, 16, 16, 256)  0           conv4_block19_1_relu[0][0]       
__________________________________________________________________________________________________
conv4_block19_2_conv (Conv2D)   (None, 14, 14, 256)  589824      conv4_block19_2_pad[0][0]        
__________________________________________________________________________________________________
conv4_block19_2_bn (BatchNormal (None, 14, 14, 256)  1024        conv4_block19_2_conv[0][0]       
__________________________________________________________________________________________________
conv4_block19_2_relu (Activatio (None, 14, 14, 256)  0           conv4_block19_2_bn[0][0]         
__________________________________________________________________________________________________
conv4_block19_3_conv (Conv2D)   (None, 14, 14, 1024) 263168      conv4_block19_2_relu[0][0]       
__________________________________________________________________________________________________
conv4_block19_out (Add)         (None, 14, 14, 1024) 0           conv4_block18_out[0][0]          
                                                                 conv4_block19_3_conv[0][0]       
__________________________________________________________________________________________________
conv4_block20_preact_bn (BatchN (None, 14, 14, 1024) 4096        conv4_block19_out[0][0]          
__________________________________________________________________________________________________
conv4_block20_preact_relu (Acti (None, 14, 14, 1024) 0           conv4_block20_preact_bn[0][0]    
__________________________________________________________________________________________________
conv4_block20_1_conv (Conv2D)   (None, 14, 14, 256)  262144      conv4_block20_preact_relu[0][0]  
__________________________________________________________________________________________________
conv4_block20_1_bn (BatchNormal (None, 14, 14, 256)  1024        conv4_block20_1_conv[0][0]       
__________________________________________________________________________________________________
conv4_block20_1_relu (Activatio (None, 14, 14, 256)  0           conv4_block20_1_bn[0][0]         
__________________________________________________________________________________________________
conv4_block20_2_pad (ZeroPaddin (None, 16, 16, 256)  0           conv4_block20_1_relu[0][0]       
__________________________________________________________________________________________________
conv4_block20_2_conv (Conv2D)   (None, 14, 14, 256)  589824      conv4_block20_2_pad[0][0]        
__________________________________________________________________________________________________
conv4_block20_2_bn (BatchNormal (None, 14, 14, 256)  1024        conv4_block20_2_conv[0][0]       
__________________________________________________________________________________________________
conv4_block20_2_relu (Activatio (None, 14, 14, 256)  0           conv4_block20_2_bn[0][0]         
__________________________________________________________________________________________________
conv4_block20_3_conv (Conv2D)   (None, 14, 14, 1024) 263168      conv4_block20_2_relu[0][0]       
__________________________________________________________________________________________________
conv4_block20_out (Add)         (None, 14, 14, 1024) 0           conv4_block19_out[0][0]          
                                                                 conv4_block20_3_conv[0][0]       
__________________________________________________________________________________________________
conv4_block21_preact_bn (BatchN (None, 14, 14, 1024) 4096        conv4_block20_out[0][0]          
__________________________________________________________________________________________________
conv4_block21_preact_relu (Acti (None, 14, 14, 1024) 0           conv4_block21_preact_bn[0][0]    
__________________________________________________________________________________________________
conv4_block21_1_conv (Conv2D)   (None, 14, 14, 256)  262144      conv4_block21_preact_relu[0][0]  
__________________________________________________________________________________________________
conv4_block21_1_bn (BatchNormal (None, 14, 14, 256)  1024        conv4_block21_1_conv[0][0]       
__________________________________________________________________________________________________
conv4_block21_1_relu (Activatio (None, 14, 14, 256)  0           conv4_block21_1_bn[0][0]         
__________________________________________________________________________________________________
conv4_block21_2_pad (ZeroPaddin (None, 16, 16, 256)  0           conv4_block21_1_relu[0][0]       
__________________________________________________________________________________________________
conv4_block21_2_conv (Conv2D)   (None, 14, 14, 256)  589824      conv4_block21_2_pad[0][0]        
__________________________________________________________________________________________________
conv4_block21_2_bn (BatchNormal (None, 14, 14, 256)  1024        conv4_block21_2_conv[0][0]       
__________________________________________________________________________________________________
conv4_block21_2_relu (Activatio (None, 14, 14, 256)  0           conv4_block21_2_bn[0][0]         
__________________________________________________________________________________________________
conv4_block21_3_conv (Conv2D)   (None, 14, 14, 1024) 263168      conv4_block21_2_relu[0][0]       
__________________________________________________________________________________________________
conv4_block21_out (Add)         (None, 14, 14, 1024) 0           conv4_block20_out[0][0]          
                                                                 conv4_block21_3_conv[0][0]       
__________________________________________________________________________________________________
conv4_block22_preact_bn (BatchN (None, 14, 14, 1024) 4096        conv4_block21_out[0][0]          
__________________________________________________________________________________________________
conv4_block22_preact_relu (Acti (None, 14, 14, 1024) 0           conv4_block22_preact_bn[0][0]    
__________________________________________________________________________________________________
conv4_block22_1_conv (Conv2D)   (None, 14, 14, 256)  262144      conv4_block22_preact_relu[0][0]  
__________________________________________________________________________________________________
conv4_block22_1_bn (BatchNormal (None, 14, 14, 256)  1024        conv4_block22_1_conv[0][0]       
__________________________________________________________________________________________________
conv4_block22_1_relu (Activatio (None, 14, 14, 256)  0           conv4_block22_1_bn[0][0]         
__________________________________________________________________________________________________
conv4_block22_2_pad (ZeroPaddin (None, 16, 16, 256)  0           conv4_block22_1_relu[0][0]       
__________________________________________________________________________________________________
conv4_block22_2_conv (Conv2D)   (None, 14, 14, 256)  589824      conv4_block22_2_pad[0][0]        
__________________________________________________________________________________________________
conv4_block22_2_bn (BatchNormal (None, 14, 14, 256)  1024        conv4_block22_2_conv[0][0]       
__________________________________________________________________________________________________
conv4_block22_2_relu (Activatio (None, 14, 14, 256)  0           conv4_block22_2_bn[0][0]         
__________________________________________________________________________________________________
conv4_block22_3_conv (Conv2D)   (None, 14, 14, 1024) 263168      conv4_block22_2_relu[0][0]       
__________________________________________________________________________________________________
conv4_block22_out (Add)         (None, 14, 14, 1024) 0           conv4_block21_out[0][0]          
                                                                 conv4_block22_3_conv[0][0]       
__________________________________________________________________________________________________
conv4_block23_preact_bn (BatchN (None, 14, 14, 1024) 4096        conv4_block22_out[0][0]          
__________________________________________________________________________________________________
conv4_block23_preact_relu (Acti (None, 14, 14, 1024) 0           conv4_block23_preact_bn[0][0]    
__________________________________________________________________________________________________
conv4_block23_1_conv (Conv2D)   (None, 14, 14, 256)  262144      conv4_block23_preact_relu[0][0]  
__________________________________________________________________________________________________
conv4_block23_1_bn (BatchNormal (None, 14, 14, 256)  1024        conv4_block23_1_conv[0][0]       
__________________________________________________________________________________________________
conv4_block23_1_relu (Activatio (None, 14, 14, 256)  0           conv4_block23_1_bn[0][0]         
__________________________________________________________________________________________________
conv4_block23_2_pad (ZeroPaddin (None, 16, 16, 256)  0           conv4_block23_1_relu[0][0]       
__________________________________________________________________________________________________
conv4_block23_2_conv (Conv2D)   (None, 14, 14, 256)  589824      conv4_block23_2_pad[0][0]        
__________________________________________________________________________________________________
conv4_block23_2_bn (BatchNormal (None, 14, 14, 256)  1024        conv4_block23_2_conv[0][0]       
__________________________________________________________________________________________________
conv4_block23_2_relu (Activatio (None, 14, 14, 256)  0           conv4_block23_2_bn[0][0]         
__________________________________________________________________________________________________
conv4_block23_3_conv (Conv2D)   (None, 14, 14, 1024) 263168      conv4_block23_2_relu[0][0]       
__________________________________________________________________________________________________
conv4_block23_out (Add)         (None, 14, 14, 1024) 0           conv4_block22_out[0][0]          
                                                                 conv4_block23_3_conv[0][0]       
__________________________________________________________________________________________________
conv4_block24_preact_bn (BatchN (None, 14, 14, 1024) 4096        conv4_block23_out[0][0]          
__________________________________________________________________________________________________
conv4_block24_preact_relu (Acti (None, 14, 14, 1024) 0           conv4_block24_preact_bn[0][0]    
__________________________________________________________________________________________________
conv4_block24_1_conv (Conv2D)   (None, 14, 14, 256)  262144      conv4_block24_preact_relu[0][0]  
__________________________________________________________________________________________________
conv4_block24_1_bn (BatchNormal (None, 14, 14, 256)  1024        conv4_block24_1_conv[0][0]       
__________________________________________________________________________________________________
conv4_block24_1_relu (Activatio (None, 14, 14, 256)  0           conv4_block24_1_bn[0][0]         
__________________________________________________________________________________________________
conv4_block24_2_pad (ZeroPaddin (None, 16, 16, 256)  0           conv4_block24_1_relu[0][0]       
__________________________________________________________________________________________________
conv4_block24_2_conv (Conv2D)   (None, 14, 14, 256)  589824      conv4_block24_2_pad[0][0]        
__________________________________________________________________________________________________
conv4_block24_2_bn (BatchNormal (None, 14, 14, 256)  1024        conv4_block24_2_conv[0][0]       
__________________________________________________________________________________________________
conv4_block24_2_relu (Activatio (None, 14, 14, 256)  0           conv4_block24_2_bn[0][0]         
__________________________________________________________________________________________________
conv4_block24_3_conv (Conv2D)   (None, 14, 14, 1024) 263168      conv4_block24_2_relu[0][0]       
__________________________________________________________________________________________________
conv4_block24_out (Add)         (None, 14, 14, 1024) 0           conv4_block23_out[0][0]          
                                                                 conv4_block24_3_conv[0][0]       
__________________________________________________________________________________________________
conv4_block25_preact_bn (BatchN (None, 14, 14, 1024) 4096        conv4_block24_out[0][0]          
__________________________________________________________________________________________________
conv4_block25_preact_relu (Acti (None, 14, 14, 1024) 0           conv4_block25_preact_bn[0][0]    
__________________________________________________________________________________________________
conv4_block25_1_conv (Conv2D)   (None, 14, 14, 256)  262144      conv4_block25_preact_relu[0][0]  
__________________________________________________________________________________________________
conv4_block25_1_bn (BatchNormal (None, 14, 14, 256)  1024        conv4_block25_1_conv[0][0]       
__________________________________________________________________________________________________
conv4_block25_1_relu (Activatio (None, 14, 14, 256)  0           conv4_block25_1_bn[0][0]         
__________________________________________________________________________________________________
conv4_block25_2_pad (ZeroPaddin (None, 16, 16, 256)  0           conv4_block25_1_relu[0][0]       
__________________________________________________________________________________________________
conv4_block25_2_conv (Conv2D)   (None, 14, 14, 256)  589824      conv4_block25_2_pad[0][0]        
__________________________________________________________________________________________________
conv4_block25_2_bn (BatchNormal (None, 14, 14, 256)  1024        conv4_block25_2_conv[0][0]       
__________________________________________________________________________________________________
conv4_block25_2_relu (Activatio (None, 14, 14, 256)  0           conv4_block25_2_bn[0][0]         
__________________________________________________________________________________________________
conv4_block25_3_conv (Conv2D)   (None, 14, 14, 1024) 263168      conv4_block25_2_relu[0][0]       
__________________________________________________________________________________________________
conv4_block25_out (Add)         (None, 14, 14, 1024) 0           conv4_block24_out[0][0]          
                                                                 conv4_block25_3_conv[0][0]       
__________________________________________________________________________________________________
conv4_block26_preact_bn (BatchN (None, 14, 14, 1024) 4096        conv4_block25_out[0][0]          
__________________________________________________________________________________________________
conv4_block26_preact_relu (Acti (None, 14, 14, 1024) 0           conv4_block26_preact_bn[0][0]    
__________________________________________________________________________________________________
conv4_block26_1_conv (Conv2D)   (None, 14, 14, 256)  262144      conv4_block26_preact_relu[0][0]  
__________________________________________________________________________________________________
conv4_block26_1_bn (BatchNormal (None, 14, 14, 256)  1024        conv4_block26_1_conv[0][0]       
__________________________________________________________________________________________________
conv4_block26_1_relu (Activatio (None, 14, 14, 256)  0           conv4_block26_1_bn[0][0]         
__________________________________________________________________________________________________
conv4_block26_2_pad (ZeroPaddin (None, 16, 16, 256)  0           conv4_block26_1_relu[0][0]       
__________________________________________________________________________________________________
conv4_block26_2_conv (Conv2D)   (None, 14, 14, 256)  589824      conv4_block26_2_pad[0][0]        
__________________________________________________________________________________________________
conv4_block26_2_bn (BatchNormal (None, 14, 14, 256)  1024        conv4_block26_2_conv[0][0]       
__________________________________________________________________________________________________
conv4_block26_2_relu (Activatio (None, 14, 14, 256)  0           conv4_block26_2_bn[0][0]         
__________________________________________________________________________________________________
conv4_block26_3_conv (Conv2D)   (None, 14, 14, 1024) 263168      conv4_block26_2_relu[0][0]       
__________________________________________________________________________________________________
conv4_block26_out (Add)         (None, 14, 14, 1024) 0           conv4_block25_out[0][0]          
                                                                 conv4_block26_3_conv[0][0]       
__________________________________________________________________________________________________
conv4_block27_preact_bn (BatchN (None, 14, 14, 1024) 4096        conv4_block26_out[0][0]          
__________________________________________________________________________________________________
conv4_block27_preact_relu (Acti (None, 14, 14, 1024) 0           conv4_block27_preact_bn[0][0]    
__________________________________________________________________________________________________
conv4_block27_1_conv (Conv2D)   (None, 14, 14, 256)  262144      conv4_block27_preact_relu[0][0]  
__________________________________________________________________________________________________
conv4_block27_1_bn (BatchNormal (None, 14, 14, 256)  1024        conv4_block27_1_conv[0][0]       
__________________________________________________________________________________________________
conv4_block27_1_relu (Activatio (None, 14, 14, 256)  0           conv4_block27_1_bn[0][0]         
__________________________________________________________________________________________________
conv4_block27_2_pad (ZeroPaddin (None, 16, 16, 256)  0           conv4_block27_1_relu[0][0]       
__________________________________________________________________________________________________
conv4_block27_2_conv (Conv2D)   (None, 14, 14, 256)  589824      conv4_block27_2_pad[0][0]        
__________________________________________________________________________________________________
conv4_block27_2_bn (BatchNormal (None, 14, 14, 256)  1024        conv4_block27_2_conv[0][0]       
__________________________________________________________________________________________________
conv4_block27_2_relu (Activatio (None, 14, 14, 256)  0           conv4_block27_2_bn[0][0]         
__________________________________________________________________________________________________
conv4_block27_3_conv (Conv2D)   (None, 14, 14, 1024) 263168      conv4_block27_2_relu[0][0]       
__________________________________________________________________________________________________
conv4_block27_out (Add)         (None, 14, 14, 1024) 0           conv4_block26_out[0][0]          
                                                                 conv4_block27_3_conv[0][0]       
__________________________________________________________________________________________________
conv4_block28_preact_bn (BatchN (None, 14, 14, 1024) 4096        conv4_block27_out[0][0]          
__________________________________________________________________________________________________
conv4_block28_preact_relu (Acti (None, 14, 14, 1024) 0           conv4_block28_preact_bn[0][0]    
__________________________________________________________________________________________________
conv4_block28_1_conv (Conv2D)   (None, 14, 14, 256)  262144      conv4_block28_preact_relu[0][0]  
__________________________________________________________________________________________________
conv4_block28_1_bn (BatchNormal (None, 14, 14, 256)  1024        conv4_block28_1_conv[0][0]       
__________________________________________________________________________________________________
conv4_block28_1_relu (Activatio (None, 14, 14, 256)  0           conv4_block28_1_bn[0][0]         
__________________________________________________________________________________________________
conv4_block28_2_pad (ZeroPaddin (None, 16, 16, 256)  0           conv4_block28_1_relu[0][0]       
__________________________________________________________________________________________________
conv4_block28_2_conv (Conv2D)   (None, 14, 14, 256)  589824      conv4_block28_2_pad[0][0]        
__________________________________________________________________________________________________
conv4_block28_2_bn (BatchNormal (None, 14, 14, 256)  1024        conv4_block28_2_conv[0][0]       
__________________________________________________________________________________________________
conv4_block28_2_relu (Activatio (None, 14, 14, 256)  0           conv4_block28_2_bn[0][0]         
__________________________________________________________________________________________________
conv4_block28_3_conv (Conv2D)   (None, 14, 14, 1024) 263168      conv4_block28_2_relu[0][0]       
__________________________________________________________________________________________________
conv4_block28_out (Add)         (None, 14, 14, 1024) 0           conv4_block27_out[0][0]          
                                                                 conv4_block28_3_conv[0][0]       
__________________________________________________________________________________________________
conv4_block29_preact_bn (BatchN (None, 14, 14, 1024) 4096        conv4_block28_out[0][0]          
__________________________________________________________________________________________________
conv4_block29_preact_relu (Acti (None, 14, 14, 1024) 0           conv4_block29_preact_bn[0][0]    
__________________________________________________________________________________________________
conv4_block29_1_conv (Conv2D)   (None, 14, 14, 256)  262144      conv4_block29_preact_relu[0][0]  
__________________________________________________________________________________________________
conv4_block29_1_bn (BatchNormal (None, 14, 14, 256)  1024        conv4_block29_1_conv[0][0]       
__________________________________________________________________________________________________
conv4_block29_1_relu (Activatio (None, 14, 14, 256)  0           conv4_block29_1_bn[0][0]         
__________________________________________________________________________________________________
conv4_block29_2_pad (ZeroPaddin (None, 16, 16, 256)  0           conv4_block29_1_relu[0][0]       
__________________________________________________________________________________________________
conv4_block29_2_conv (Conv2D)   (None, 14, 14, 256)  589824      conv4_block29_2_pad[0][0]        
__________________________________________________________________________________________________
conv4_block29_2_bn (BatchNormal (None, 14, 14, 256)  1024        conv4_block29_2_conv[0][0]       
__________________________________________________________________________________________________
conv4_block29_2_relu (Activatio (None, 14, 14, 256)  0           conv4_block29_2_bn[0][0]         
__________________________________________________________________________________________________
conv4_block29_3_conv (Conv2D)   (None, 14, 14, 1024) 263168      conv4_block29_2_relu[0][0]       
__________________________________________________________________________________________________
conv4_block29_out (Add)         (None, 14, 14, 1024) 0           conv4_block28_out[0][0]          
                                                                 conv4_block29_3_conv[0][0]       
__________________________________________________________________________________________________
conv4_block30_preact_bn (BatchN (None, 14, 14, 1024) 4096        conv4_block29_out[0][0]          
__________________________________________________________________________________________________
conv4_block30_preact_relu (Acti (None, 14, 14, 1024) 0           conv4_block30_preact_bn[0][0]    
__________________________________________________________________________________________________
conv4_block30_1_conv (Conv2D)   (None, 14, 14, 256)  262144      conv4_block30_preact_relu[0][0]  
__________________________________________________________________________________________________
conv4_block30_1_bn (BatchNormal (None, 14, 14, 256)  1024        conv4_block30_1_conv[0][0]       
__________________________________________________________________________________________________
conv4_block30_1_relu (Activatio (None, 14, 14, 256)  0           conv4_block30_1_bn[0][0]         
__________________________________________________________________________________________________
conv4_block30_2_pad (ZeroPaddin (None, 16, 16, 256)  0           conv4_block30_1_relu[0][0]       
__________________________________________________________________________________________________
conv4_block30_2_conv (Conv2D)   (None, 14, 14, 256)  589824      conv4_block30_2_pad[0][0]        
__________________________________________________________________________________________________
conv4_block30_2_bn (BatchNormal (None, 14, 14, 256)  1024        conv4_block30_2_conv[0][0]       
__________________________________________________________________________________________________
conv4_block30_2_relu (Activatio (None, 14, 14, 256)  0           conv4_block30_2_bn[0][0]         
__________________________________________________________________________________________________
conv4_block30_3_conv (Conv2D)   (None, 14, 14, 1024) 263168      conv4_block30_2_relu[0][0]       
__________________________________________________________________________________________________
conv4_block30_out (Add)         (None, 14, 14, 1024) 0           conv4_block29_out[0][0]          
                                                                 conv4_block30_3_conv[0][0]       
__________________________________________________________________________________________________
conv4_block31_preact_bn (BatchN (None, 14, 14, 1024) 4096        conv4_block30_out[0][0]          
__________________________________________________________________________________________________
conv4_block31_preact_relu (Acti (None, 14, 14, 1024) 0           conv4_block31_preact_bn[0][0]    
__________________________________________________________________________________________________
conv4_block31_1_conv (Conv2D)   (None, 14, 14, 256)  262144      conv4_block31_preact_relu[0][0]  
__________________________________________________________________________________________________
conv4_block31_1_bn (BatchNormal (None, 14, 14, 256)  1024        conv4_block31_1_conv[0][0]       
__________________________________________________________________________________________________
conv4_block31_1_relu (Activatio (None, 14, 14, 256)  0           conv4_block31_1_bn[0][0]         
__________________________________________________________________________________________________
conv4_block31_2_pad (ZeroPaddin (None, 16, 16, 256)  0           conv4_block31_1_relu[0][0]       
__________________________________________________________________________________________________
conv4_block31_2_conv (Conv2D)   (None, 14, 14, 256)  589824      conv4_block31_2_pad[0][0]        
__________________________________________________________________________________________________
conv4_block31_2_bn (BatchNormal (None, 14, 14, 256)  1024        conv4_block31_2_conv[0][0]       
__________________________________________________________________________________________________
conv4_block31_2_relu (Activatio (None, 14, 14, 256)  0           conv4_block31_2_bn[0][0]         
__________________________________________________________________________________________________
conv4_block31_3_conv (Conv2D)   (None, 14, 14, 1024) 263168      conv4_block31_2_relu[0][0]       
__________________________________________________________________________________________________
conv4_block31_out (Add)         (None, 14, 14, 1024) 0           conv4_block30_out[0][0]          
                                                                 conv4_block31_3_conv[0][0]       
__________________________________________________________________________________________________
conv4_block32_preact_bn (BatchN (None, 14, 14, 1024) 4096        conv4_block31_out[0][0]          
__________________________________________________________________________________________________
conv4_block32_preact_relu (Acti (None, 14, 14, 1024) 0           conv4_block32_preact_bn[0][0]    
__________________________________________________________________________________________________
conv4_block32_1_conv (Conv2D)   (None, 14, 14, 256)  262144      conv4_block32_preact_relu[0][0]  
__________________________________________________________________________________________________
conv4_block32_1_bn (BatchNormal (None, 14, 14, 256)  1024        conv4_block32_1_conv[0][0]       
__________________________________________________________________________________________________
conv4_block32_1_relu (Activatio (None, 14, 14, 256)  0           conv4_block32_1_bn[0][0]         
__________________________________________________________________________________________________
conv4_block32_2_pad (ZeroPaddin (None, 16, 16, 256)  0           conv4_block32_1_relu[0][0]       
__________________________________________________________________________________________________
conv4_block32_2_conv (Conv2D)   (None, 14, 14, 256)  589824      conv4_block32_2_pad[0][0]        
__________________________________________________________________________________________________
conv4_block32_2_bn (BatchNormal (None, 14, 14, 256)  1024        conv4_block32_2_conv[0][0]       
__________________________________________________________________________________________________
conv4_block32_2_relu (Activatio (None, 14, 14, 256)  0           conv4_block32_2_bn[0][0]         
__________________________________________________________________________________________________
conv4_block32_3_conv (Conv2D)   (None, 14, 14, 1024) 263168      conv4_block32_2_relu[0][0]       
__________________________________________________________________________________________________
conv4_block32_out (Add)         (None, 14, 14, 1024) 0           conv4_block31_out[0][0]          
                                                                 conv4_block32_3_conv[0][0]       
__________________________________________________________________________________________________
conv4_block33_preact_bn (BatchN (None, 14, 14, 1024) 4096        conv4_block32_out[0][0]          
__________________________________________________________________________________________________
conv4_block33_preact_relu (Acti (None, 14, 14, 1024) 0           conv4_block33_preact_bn[0][0]    
__________________________________________________________________________________________________
conv4_block33_1_conv (Conv2D)   (None, 14, 14, 256)  262144      conv4_block33_preact_relu[0][0]  
__________________________________________________________________________________________________
conv4_block33_1_bn (BatchNormal (None, 14, 14, 256)  1024        conv4_block33_1_conv[0][0]       
__________________________________________________________________________________________________
conv4_block33_1_relu (Activatio (None, 14, 14, 256)  0           conv4_block33_1_bn[0][0]         
__________________________________________________________________________________________________
conv4_block33_2_pad (ZeroPaddin (None, 16, 16, 256)  0           conv4_block33_1_relu[0][0]       
__________________________________________________________________________________________________
conv4_block33_2_conv (Conv2D)   (None, 14, 14, 256)  589824      conv4_block33_2_pad[0][0]        
__________________________________________________________________________________________________
conv4_block33_2_bn (BatchNormal (None, 14, 14, 256)  1024        conv4_block33_2_conv[0][0]       
__________________________________________________________________________________________________
conv4_block33_2_relu (Activatio (None, 14, 14, 256)  0           conv4_block33_2_bn[0][0]         
__________________________________________________________________________________________________
conv4_block33_3_conv (Conv2D)   (None, 14, 14, 1024) 263168      conv4_block33_2_relu[0][0]       
__________________________________________________________________________________________________
conv4_block33_out (Add)         (None, 14, 14, 1024) 0           conv4_block32_out[0][0]          
                                                                 conv4_block33_3_conv[0][0]       
__________________________________________________________________________________________________
conv4_block34_preact_bn (BatchN (None, 14, 14, 1024) 4096        conv4_block33_out[0][0]          
__________________________________________________________________________________________________
conv4_block34_preact_relu (Acti (None, 14, 14, 1024) 0           conv4_block34_preact_bn[0][0]    
__________________________________________________________________________________________________
conv4_block34_1_conv (Conv2D)   (None, 14, 14, 256)  262144      conv4_block34_preact_relu[0][0]  
__________________________________________________________________________________________________
conv4_block34_1_bn (BatchNormal (None, 14, 14, 256)  1024        conv4_block34_1_conv[0][0]       
__________________________________________________________________________________________________
conv4_block34_1_relu (Activatio (None, 14, 14, 256)  0           conv4_block34_1_bn[0][0]         
__________________________________________________________________________________________________
conv4_block34_2_pad (ZeroPaddin (None, 16, 16, 256)  0           conv4_block34_1_relu[0][0]       
__________________________________________________________________________________________________
conv4_block34_2_conv (Conv2D)   (None, 14, 14, 256)  589824      conv4_block34_2_pad[0][0]        
__________________________________________________________________________________________________
conv4_block34_2_bn (BatchNormal (None, 14, 14, 256)  1024        conv4_block34_2_conv[0][0]       
__________________________________________________________________________________________________
conv4_block34_2_relu (Activatio (None, 14, 14, 256)  0           conv4_block34_2_bn[0][0]         
__________________________________________________________________________________________________
conv4_block34_3_conv (Conv2D)   (None, 14, 14, 1024) 263168      conv4_block34_2_relu[0][0]       
__________________________________________________________________________________________________
conv4_block34_out (Add)         (None, 14, 14, 1024) 0           conv4_block33_out[0][0]          
                                                                 conv4_block34_3_conv[0][0]       
__________________________________________________________________________________________________
conv4_block35_preact_bn (BatchN (None, 14, 14, 1024) 4096        conv4_block34_out[0][0]          
__________________________________________________________________________________________________
conv4_block35_preact_relu (Acti (None, 14, 14, 1024) 0           conv4_block35_preact_bn[0][0]    
__________________________________________________________________________________________________
conv4_block35_1_conv (Conv2D)   (None, 14, 14, 256)  262144      conv4_block35_preact_relu[0][0]  
__________________________________________________________________________________________________
conv4_block35_1_bn (BatchNormal (None, 14, 14, 256)  1024        conv4_block35_1_conv[0][0]       
__________________________________________________________________________________________________
conv4_block35_1_relu (Activatio (None, 14, 14, 256)  0           conv4_block35_1_bn[0][0]         
__________________________________________________________________________________________________
conv4_block35_2_pad (ZeroPaddin (None, 16, 16, 256)  0           conv4_block35_1_relu[0][0]       
__________________________________________________________________________________________________
conv4_block35_2_conv (Conv2D)   (None, 14, 14, 256)  589824      conv4_block35_2_pad[0][0]        
__________________________________________________________________________________________________
conv4_block35_2_bn (BatchNormal (None, 14, 14, 256)  1024        conv4_block35_2_conv[0][0]       
__________________________________________________________________________________________________
conv4_block35_2_relu (Activatio (None, 14, 14, 256)  0           conv4_block35_2_bn[0][0]         
__________________________________________________________________________________________________
conv4_block35_3_conv (Conv2D)   (None, 14, 14, 1024) 263168      conv4_block35_2_relu[0][0]       
__________________________________________________________________________________________________
conv4_block35_out (Add)         (None, 14, 14, 1024) 0           conv4_block34_out[0][0]          
                                                                 conv4_block35_3_conv[0][0]       
__________________________________________________________________________________________________
conv4_block36_preact_bn (BatchN (None, 14, 14, 1024) 4096        conv4_block35_out[0][0]          
__________________________________________________________________________________________________
conv4_block36_preact_relu (Acti (None, 14, 14, 1024) 0           conv4_block36_preact_bn[0][0]    
__________________________________________________________________________________________________
conv4_block36_1_conv (Conv2D)   (None, 14, 14, 256)  262144      conv4_block36_preact_relu[0][0]  
__________________________________________________________________________________________________
conv4_block36_1_bn (BatchNormal (None, 14, 14, 256)  1024        conv4_block36_1_conv[0][0]       
__________________________________________________________________________________________________
conv4_block36_1_relu (Activatio (None, 14, 14, 256)  0           conv4_block36_1_bn[0][0]         
__________________________________________________________________________________________________
conv4_block36_2_pad (ZeroPaddin (None, 16, 16, 256)  0           conv4_block36_1_relu[0][0]       
__________________________________________________________________________________________________
conv4_block36_2_conv (Conv2D)   (None, 7, 7, 256)    589824      conv4_block36_2_pad[0][0]        
__________________________________________________________________________________________________
conv4_block36_2_bn (BatchNormal (None, 7, 7, 256)    1024        conv4_block36_2_conv[0][0]       
__________________________________________________________________________________________________
conv4_block36_2_relu (Activatio (None, 7, 7, 256)    0           conv4_block36_2_bn[0][0]         
__________________________________________________________________________________________________
max_pooling2d_19 (MaxPooling2D) (None, 7, 7, 1024)   0           conv4_block35_out[0][0]          
__________________________________________________________________________________________________
conv4_block36_3_conv (Conv2D)   (None, 7, 7, 1024)   263168      conv4_block36_2_relu[0][0]       
__________________________________________________________________________________________________
conv4_block36_out (Add)         (None, 7, 7, 1024)   0           max_pooling2d_19[0][0]           
                                                                 conv4_block36_3_conv[0][0]       
__________________________________________________________________________________________________
conv5_block1_preact_bn (BatchNo (None, 7, 7, 1024)   4096        conv4_block36_out[0][0]          
__________________________________________________________________________________________________
conv5_block1_preact_relu (Activ (None, 7, 7, 1024)   0           conv5_block1_preact_bn[0][0]     
__________________________________________________________________________________________________
conv5_block1_1_conv (Conv2D)    (None, 7, 7, 512)    524288      conv5_block1_preact_relu[0][0]   
__________________________________________________________________________________________________
conv5_block1_1_bn (BatchNormali (None, 7, 7, 512)    2048        conv5_block1_1_conv[0][0]        
__________________________________________________________________________________________________
conv5_block1_1_relu (Activation (None, 7, 7, 512)    0           conv5_block1_1_bn[0][0]          
__________________________________________________________________________________________________
conv5_block1_2_pad (ZeroPadding (None, 9, 9, 512)    0           conv5_block1_1_relu[0][0]        
__________________________________________________________________________________________________
conv5_block1_2_conv (Conv2D)    (None, 7, 7, 512)    2359296     conv5_block1_2_pad[0][0]         
__________________________________________________________________________________________________
conv5_block1_2_bn (BatchNormali (None, 7, 7, 512)    2048        conv5_block1_2_conv[0][0]        
__________________________________________________________________________________________________
conv5_block1_2_relu (Activation (None, 7, 7, 512)    0           conv5_block1_2_bn[0][0]          
__________________________________________________________________________________________________
conv5_block1_0_conv (Conv2D)    (None, 7, 7, 2048)   2099200     conv5_block1_preact_relu[0][0]   
__________________________________________________________________________________________________
conv5_block1_3_conv (Conv2D)    (None, 7, 7, 2048)   1050624     conv5_block1_2_relu[0][0]        
__________________________________________________________________________________________________
conv5_block1_out (Add)          (None, 7, 7, 2048)   0           conv5_block1_0_conv[0][0]        
                                                                 conv5_block1_3_conv[0][0]        
__________________________________________________________________________________________________
conv5_block2_preact_bn (BatchNo (None, 7, 7, 2048)   8192        conv5_block1_out[0][0]           
__________________________________________________________________________________________________
conv5_block2_preact_relu (Activ (None, 7, 7, 2048)   0           conv5_block2_preact_bn[0][0]     
__________________________________________________________________________________________________
conv5_block2_1_conv (Conv2D)    (None, 7, 7, 512)    1048576     conv5_block2_preact_relu[0][0]   
__________________________________________________________________________________________________
conv5_block2_1_bn (BatchNormali (None, 7, 7, 512)    2048        conv5_block2_1_conv[0][0]        
__________________________________________________________________________________________________
conv5_block2_1_relu (Activation (None, 7, 7, 512)    0           conv5_block2_1_bn[0][0]          
__________________________________________________________________________________________________
conv5_block2_2_pad (ZeroPadding (None, 9, 9, 512)    0           conv5_block2_1_relu[0][0]        
__________________________________________________________________________________________________
conv5_block2_2_conv (Conv2D)    (None, 7, 7, 512)    2359296     conv5_block2_2_pad[0][0]         
__________________________________________________________________________________________________
conv5_block2_2_bn (BatchNormali (None, 7, 7, 512)    2048        conv5_block2_2_conv[0][0]        
__________________________________________________________________________________________________
conv5_block2_2_relu (Activation (None, 7, 7, 512)    0           conv5_block2_2_bn[0][0]          
__________________________________________________________________________________________________
conv5_block2_3_conv (Conv2D)    (None, 7, 7, 2048)   1050624     conv5_block2_2_relu[0][0]        
__________________________________________________________________________________________________
conv5_block2_out (Add)          (None, 7, 7, 2048)   0           conv5_block1_out[0][0]           
                                                                 conv5_block2_3_conv[0][0]        
__________________________________________________________________________________________________
conv5_block3_preact_bn (BatchNo (None, 7, 7, 2048)   8192        conv5_block2_out[0][0]           
__________________________________________________________________________________________________
conv5_block3_preact_relu (Activ (None, 7, 7, 2048)   0           conv5_block3_preact_bn[0][0]     
__________________________________________________________________________________________________
conv5_block3_1_conv (Conv2D)    (None, 7, 7, 512)    1048576     conv5_block3_preact_relu[0][0]   
__________________________________________________________________________________________________
conv5_block3_1_bn (BatchNormali (None, 7, 7, 512)    2048        conv5_block3_1_conv[0][0]        
__________________________________________________________________________________________________
conv5_block3_1_relu (Activation (None, 7, 7, 512)    0           conv5_block3_1_bn[0][0]          
__________________________________________________________________________________________________
conv5_block3_2_pad (ZeroPadding (None, 9, 9, 512)    0           conv5_block3_1_relu[0][0]        
__________________________________________________________________________________________________
conv5_block3_2_conv (Conv2D)    (None, 7, 7, 512)    2359296     conv5_block3_2_pad[0][0]         
__________________________________________________________________________________________________
conv5_block3_2_bn (BatchNormali (None, 7, 7, 512)    2048        conv5_block3_2_conv[0][0]        
__________________________________________________________________________________________________
conv5_block3_2_relu (Activation (None, 7, 7, 512)    0           conv5_block3_2_bn[0][0]          
__________________________________________________________________________________________________
conv5_block3_3_conv (Conv2D)    (None, 7, 7, 2048)   1050624     conv5_block3_2_relu[0][0]        
__________________________________________________________________________________________________
conv5_block3_out (Add)          (None, 7, 7, 2048)   0           conv5_block2_out[0][0]           
                                                                 conv5_block3_3_conv[0][0]        
__________________________________________________________________________________________________
post_bn (BatchNormalization)    (None, 7, 7, 2048)   8192        conv5_block3_out[0][0]           
__________________________________________________________________________________________________
post_relu (Activation)          (None, 7, 7, 2048)   0           post_bn[0][0]                    
__________________________________________________________________________________________________
global_average_pooling2d_1 (Glo (None, 2048)         0           post_relu[0][0]                  
__________________________________________________________________________________________________
dense_14 (Dense)                (None, 1000)         2049000     global_average_pooling2d_1[0][0] 
__________________________________________________________________________________________________
dense_15 (Dense)                (None, 10)           10010       dense_14[0][0]                   
==================================================================================================
Total params: 60,390,658
Trainable params: 56,430,338
Non-trainable params: 3,960,320
__________________________________________________________________________________________________
In [122]:
model.compile(loss='categorical_crossentropy', # 'categorical_crossentropy' 'mean_squared_error'
                optimizer='rmsprop', # 'adadelta' 'rmsprop'
                metrics=['accuracy'])
In [179]:
BS  = 16
epochs = 10
history2 = model.fit_generator(datagen_train1.flow(X_train1, Y_train1, batch_size=BS), 
                   steps_per_epoch=len(X_train)//BS, # how many generators to go through per epoch
                   epochs=epochs, verbose=1,
                   validation_data=datagen_val1.flow(X_val1,Y_val1, batch_size=BS),
                   validation_steps=len(X_val)//BS,
                   callbacks=[EarlyStopping(monitor='val_loss', patience=3)],
                  )
/users/sbataju/.conda/envs/tf_gpu/lib/python3.9/site-packages/keras/engine/training.py:1972: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
  warnings.warn('`Model.fit_generator` is deprecated and '
Epoch 1/10
562/562 [==============================] - 138s 245ms/step - loss: 0.4256 - accuracy: 0.8660 - val_loss: 0.3193 - val_accuracy: 0.8841
Epoch 2/10
562/562 [==============================] - 138s 245ms/step - loss: 0.3659 - accuracy: 0.8795 - val_loss: 0.2875 - val_accuracy: 0.9032
Epoch 3/10
562/562 [==============================] - 140s 248ms/step - loss: 0.3395 - accuracy: 0.8907 - val_loss: 0.3440 - val_accuracy: 0.8931
Epoch 4/10
562/562 [==============================] - 137s 243ms/step - loss: 0.3078 - accuracy: 0.9030 - val_loss: 0.3540 - val_accuracy: 0.9123
Epoch 5/10
562/562 [==============================] - 138s 245ms/step - loss: 0.2864 - accuracy: 0.9135 - val_loss: 0.5100 - val_accuracy: 0.8851
In [180]:
plot_history(history2)
yhat = np.argmax(model.predict(x_test1), axis=1)
acc = mt.accuracy_score(y_test1,yhat)
cm = mt.confusion_matrix(y_test1,yhat)
plot_confusion_matrix(cm,list(label.values()),title= f"CM Test Acc:{acc:0.3f}")
In [181]:
# X_train1, Y_train1,
yhat = np.argmax(model.predict(X_train1), axis=1)
acc = mt.accuracy_score(np.argmax(Y_train1, axis=1),yhat)
cm = mt.confusion_matrix(np.argmax(Y_train1, axis=1),yhat)
plot_confusion_matrix(cm,list(label.values()),title= f"CM Train Acc:{acc:0.3f}")
In [182]:
# X_train1, Y_train1,
yhat = np.argmax(model.predict(X_val1), axis=1)
acc = mt.accuracy_score(np.argmax(Y_val1, axis=1),yhat)
cm = mt.confusion_matrix(np.argmax(Y_val1, axis=1),yhat)
plot_confusion_matrix(cm,list(label.values()),title= f"CM Val Acc:{acc:0.3f}")
Not the best result but it is working.


Deep Nural Network


In [177]:
%%time 

dnn = Sequential()
# add one layer on flattened output
# cnn.add(Dropout(0.25)) # add some dropout for regularization after conv layers
dnn.add(Flatten(input_shape = (img_wh,img_wh,3)))
# dnn.add(Dropout(0.25))
dnn.add(Dense(1024, activation='relu'))
# dnn.add(Dropout(0.25))
dnn.add(Dense(512, activation='relu'))

dnn.add(Dense(268, activation='relu'))
# dnn.add(Dropout(0.5))
dnn.add(Dense(128, activation='relu'))
# dnn.add(Dropout(0.5)) # add some dropout for regularization, again!
dnn.add(Dense(NUM_CLASSES, activation='softmax'))

# Let's train the model 
dnn.compile(loss='categorical_crossentropy', # 'categorical_crossentropy' 'mean_squared_error'
              optimizer='rmsprop', # 'adadelta' 'rmsprop'
              metrics=['accuracy'])
dnn.summary()
Model: "sequential_7"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
flatten_9 (Flatten)          (None, 49152)             0         
_________________________________________________________________
dense_24 (Dense)             (None, 1024)              50332672  
_________________________________________________________________
dense_25 (Dense)             (None, 512)               524800    
_________________________________________________________________
dense_26 (Dense)             (None, 268)               137484    
_________________________________________________________________
dense_27 (Dense)             (None, 128)               34432     
_________________________________________________________________
dense_28 (Dense)             (None, 10)                1290      
=================================================================
Total params: 51,030,678
Trainable params: 51,030,678
Non-trainable params: 0
_________________________________________________________________
CPU times: user 72.1 ms, sys: 7.02 ms, total: 79.1 ms
Wall time: 75.4 ms
In [178]:
BS  = 32
epochs = 20
historyd = dnn.fit_generator(datagen_train.flow(X_train, Y_train, batch_size=BS), 
                   steps_per_epoch=len(X_train)//BS, # how many generators to go through per epoch
                   epochs=epochs, verbose=1,
                   validation_data=datagen_val.flow(X_val,Y_val, batch_size=BS),
                   validation_steps=len(X_val)//BS,
                   callbacks=[EarlyStopping(monitor='val_loss', patience=3)],
                  )
Epoch 1/20
281/281 [==============================] - 51s 181ms/step - loss: 6.2598 - accuracy: 0.1063 - val_loss: 2.2534 - val_accuracy: 0.1502
Epoch 2/20
281/281 [==============================] - 50s 177ms/step - loss: 2.3314 - accuracy: 0.1182 - val_loss: 2.2715 - val_accuracy: 0.1270
Epoch 3/20
281/281 [==============================] - 50s 178ms/step - loss: 2.3031 - accuracy: 0.1385 - val_loss: 2.2178 - val_accuracy: 0.1532
Epoch 4/20
281/281 [==============================] - 49s 175ms/step - loss: 2.2861 - accuracy: 0.1276 - val_loss: 2.3692 - val_accuracy: 0.1109
Epoch 5/20
281/281 [==============================] - 47s 168ms/step - loss: 2.2696 - accuracy: 0.1461 - val_loss: 2.1914 - val_accuracy: 0.1633
Epoch 6/20
281/281 [==============================] - 50s 179ms/step - loss: 2.2744 - accuracy: 0.1455 - val_loss: 2.1855 - val_accuracy: 0.1976
Epoch 7/20
281/281 [==============================] - 50s 179ms/step - loss: 2.2306 - accuracy: 0.1703 - val_loss: 2.1497 - val_accuracy: 0.1895
Epoch 8/20
281/281 [==============================] - 51s 180ms/step - loss: 2.1498 - accuracy: 0.2056 - val_loss: 2.0988 - val_accuracy: 0.1895
Epoch 9/20
281/281 [==============================] - 50s 179ms/step - loss: 2.1083 - accuracy: 0.2092 - val_loss: 2.0389 - val_accuracy: 0.2319
Epoch 10/20
281/281 [==============================] - 49s 175ms/step - loss: 2.0614 - accuracy: 0.2261 - val_loss: 1.9898 - val_accuracy: 0.2752
Epoch 11/20
281/281 [==============================] - 50s 179ms/step - loss: 2.0062 - accuracy: 0.2645 - val_loss: 1.9162 - val_accuracy: 0.2853
Epoch 12/20
281/281 [==============================] - 50s 179ms/step - loss: 1.9387 - accuracy: 0.2855 - val_loss: 1.8815 - val_accuracy: 0.3145
Epoch 13/20
281/281 [==============================] - 50s 177ms/step - loss: 1.9339 - accuracy: 0.3004 - val_loss: 1.8111 - val_accuracy: 0.3266
Epoch 14/20
281/281 [==============================] - 50s 177ms/step - loss: 1.8926 - accuracy: 0.3147 - val_loss: 1.8222 - val_accuracy: 0.3196
Epoch 15/20
281/281 [==============================] - 50s 178ms/step - loss: 1.8744 - accuracy: 0.3199 - val_loss: 2.1506 - val_accuracy: 0.3185
Epoch 16/20
281/281 [==============================] - 50s 179ms/step - loss: 1.8359 - accuracy: 0.3349 - val_loss: 1.9936 - val_accuracy: 0.3306
In [181]:
plot_history(historyd)
plt.show()

yhat = np.argmax(dnn.predict(x_test), axis=1)
acc = mt.accuracy_score(y_test,yhat)
cm = mt.confusion_matrix(y_test,yhat)
plot_confusion_matrix(cm,list(label.values()),title= f"CM Test Acc:{acc}")
plt.show()
# X_train1, Y_train1,
yhat = np.argmax(dnn.predict(X_train), axis=1)
acc = mt.accuracy_score(np.argmax(Y_train, axis=1),yhat)
cm = mt.confusion_matrix(np.argmax(Y_train, axis=1),yhat)
plot_confusion_matrix(cm,list(label.values()),title= f"CM Train Acc:{acc:0.3f}")
plt.show()

# X_train1, Y_train1,
yhat = np.argmax(dnn.predict(X_val), axis=1)
acc = mt.accuracy_score(np.argmax(Y_val, axis=1),yhat)
cm = mt.confusion_matrix(np.argmax(Y_val, axis=1),yhat)
plot_confusion_matrix(cm,list(label.values()),title= f"CM Val Acc:{acc:0.3f}")
plt.show()
Comparing Deep Nural Network with the AlexNet + ResNet at 15 epoch the CNN is doing better than DNN, showing CNN are better for image classification. Althought, I should have looked at more epoch for DNN. 

Conclusion

In this Lab, I have tried to first test the example CNNs in the class notebook. I also looked at changing the drop out rate in ResNet, then couple of Xception style networks were tested. After increasing the size of the images and including Batch Normalization, AlexNet+ResNet was tested giving the best result. It can be also be concluded that Batch Normalization helps with underfitting network. 
I also tested transfer learning from the top convolution network from AlexNet+ResNet and decreased the dense layer at the bottom, this hurt the accuracy slightly. And, lastly I used transfer learning on the top part of ResNet152V2 with good result. 


Citation

[1] https://www.kaggle.com/datasets/kaustubhb999/tomatoleaf
[2] 2012. Proceedings of the 25th International Conference on Neural Information Processing Systems - Volume 1. Curran Associates Inc., Red Hook, NY, USA. https://proceedings.neurips.cc/paper/2012/file/c399862d3b9d6b76c8436e924a68c45b-Paper.pdf


Hyperparamet Optmization Code

Below is the code discussed in CNN with Batch Normalization Optmization.

In [ ]:
import optuna
from optuna.visualization import plot_contour
from optuna.visualization import plot_edf
from optuna.visualization import plot_intermediate_values
from optuna.visualization import plot_optimization_history
from optuna.visualization import plot_parallel_coordinate
from optuna.visualization import plot_param_importances
from optuna.visualization import plot_slice
import tensorflow as tf

import os

from tensorflow.keras.layers import Add, Input
from tensorflow.keras.layers import average, concatenate
from tensorflow.keras.models import Model
from tensorflow.keras.regularizers import l2

import cv2
from matplotlib import pyplot as plt
import numpy as np
import os
from sklearn import metrics as mt
import matplotlib.image as mpimg
from keras.models import Sequential
from keras.layers.convolutional import Conv2D
from keras.layers.convolutional import MaxPooling2D
from keras.layers.core import Activation, Flatten, Dropout, Dense
from tensorflow.keras.layers import BatchNormalization
from sklearn.model_selection import train_test_split
from keras.preprocessing.image import ImageDataGenerator
from sklearn.preprocessing import LabelBinarizer
# import seaborn as sns
from skimage.io import imshow
from sklearn.metrics import plot_confusion_matrix
from tensorflow.keras.utils import plot_model
from skimage.transform import resize
# import pandas as pd

import tensorflow.keras as keras
from tensorflow.keras import layers
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Reshape, Input
from tensorflow.keras.layers import Dense, Dropout, Activation, Flatten
from tensorflow.keras.layers import Conv2D, MaxPooling2D
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.callbacks import EarlyStopping
from tensorflow.keras.regularizers import l2
from tensorflow.keras.layers import average 
from tensorflow.keras.models import  Model
from PIL import Image


plt.style.use('ggplot')


print("Num of GPUs available: ", len(tf.test.gpu_device_name()))

def img_to_np(DIR,flatten=True,newsize = (128,128)):
    img = Image.open(DIR)
    img = img.resize(newsize)
    return np.asarray(img)
    
    
def build_net1(img_wh,learning_rate,beta_1,beta_2,epsilon,l2_lambda,dropout):
    input_holder = Input(shape=(img_wh, img_wh, 3))

    # start with a conv layer
    x = Conv2D(filters=32,
                   input_shape = (img_wh,img_wh,1),
                   kernel_size=(3,3),
                   kernel_initializer='he_uniform', 
                   kernel_regularizer=l2(l2_lambda),
                   padding='same', 
                   activation='relu', 
                   data_format="channels_last")(input_holder)
    x = layers.BatchNormalization(axis=-1)(x)

    x = MaxPooling2D(pool_size=(2, 2), data_format="channels_last")(x)

    x = Conv2D(filters=32,
                   kernel_size=(3,3),
                   kernel_initializer='he_uniform', 
                   kernel_regularizer=l2(l2_lambda),
                   padding='same', 
                   activation='relu', 
                   data_format="channels_last")(x)
    x = layers.BatchNormalization(axis=-1)(x)

    x_split = MaxPooling2D(pool_size=(2, 2), data_format="channels_last")(x)

    x = Conv2D(filters=64,
                   kernel_size=(1,1),
                   kernel_initializer='he_uniform', 
                   kernel_regularizer=l2(l2_lambda),
                   padding='same', 
                   activation='LeakyReLU', 
                   data_format="channels_last")(x_split)
    x = layers.BatchNormalization(axis=-1)(x)

    x = Conv2D(filters=64,
                   kernel_size=(3,3),
                   kernel_initializer='he_uniform', 
                   kernel_regularizer=l2(l2_lambda),
                   padding='same', 
                   activation='LeakyReLU', 
                   data_format="channels_last")(x)
    x = layers.BatchNormalization(axis=-1)(x)

    x = Conv2D(filters=32,
                   kernel_size=(1,1),
                   kernel_initializer='he_uniform', 
                   kernel_regularizer=l2(l2_lambda),
                   padding='same', 
                   activation='LeakyReLU', 
                   data_format="channels_last")(x)
    x = layers.BatchNormalization(axis=-1)(x)

    # now add back in the split layer, x_split (residual added in)
    x = Add()([x, x_split])

    x = Activation("LeakyReLU")(x)

    x = MaxPooling2D(pool_size=(2, 2), data_format="channels_last")(x)

    x = Flatten()(x)
    x = Dropout(dropout)(x)
    x = Dense(256)(x)
    x = Activation("relu")(x)
    x = Dropout(dropout)(x)
    x = Dense(NUM_CLASSES)(x)
    x = Activation('softmax')(x)

    resnet_batch1 = Model(inputs=input_holder,outputs=x)
    resnet_batch1.compile(loss='categorical_crossentropy', # 'categorical_crossentropy' 'mean_squared_error'
                          optimizer=tf.keras.optimizers.Adam(
                                                learning_rate=learning_rate,
                                                beta_1=beta_1,
                                                beta_2=beta_2,
                                                epsilon=epsilon,
#                                                 amsgrad=False,
                                                name="Adam",), # 'adadelta' 'rmsprop'
                          metrics=['accuracy'])
    return resnet_batch1


TEST_DIR="/users/sbataju/CS5324/tomato/val/"

label={}
index=0
for FOLDER in os.listdir(TEST_DIR):
#     print(FOLDER,index)
    label[index]=FOLDER
    index=index+1
inv_label = {v: k for k, v in label.items()}

NUM_CLASSES = 10
index=0
data=[]
for FOLDER in os.listdir(TEST_DIR):
    print(FOLDER,inv_label[FOLDER])
    for image_dir in os.listdir(TEST_DIR+FOLDER):
        data.append({"x":img_to_np(TEST_DIR+FOLDER+"/"+image_dir,flatten=False,newsize=(224,224)),"y":label[inv_label[FOLDER]]})
#         assert False
    index=index+1
    
x,y=[],[]
for obj in data:
    x.append(obj["x"])
    y.append(obj["y"])
x_test1 = np.array(x)
y_test1 = np.array([inv_label[i] for i in y],dtype=np.float16)  





TRAIN_DIR = "/users/sbataju/CS5324/tomato/train/"
index=0
data=[]
for FOLDER in os.listdir(TRAIN_DIR):
    print(FOLDER,'  ',inv_label[FOLDER])
    for image_dir in os.listdir(TRAIN_DIR+FOLDER):
        data.append({"x":img_to_np(TRAIN_DIR+FOLDER+"/"+image_dir,flatten=False,newsize=(224,224)),"y":label[inv_label[FOLDER]]})
    index=index+1
x,y=[],[]
for obj in data:
    x.append(obj["x"])
    y.append(obj["y"])
x_train1 = np.array(x)
y_train1 = np.array([inv_label[i] for i in y],dtype=np.float16)  

y_train_ohe1 = keras.utils.to_categorical(y_train1, NUM_CLASSES)
y_test_ohe1 = keras.utils.to_categorical(y_test1, NUM_CLASSES)

from sklearn.model_selection import train_test_split
X_train1, X_val1, Y_train1, Y_val1 = train_test_split(
    x_train1, y_train_ohe1, test_size=0.1, random_state=42,stratify=y_train1)



X_train1,X_val1 =X_train1/255.0 , X_val1/255.0 
x_test1 = x_test1/255.0


datagen_train1 = ImageDataGenerator(
    featurewise_center=False,
    samplewise_center=False,
    featurewise_std_normalization=False,
    samplewise_std_normalization=False,
    zca_whitening=False,
    rotation_range=180, # used, Int. Degree range for random rotations.
    width_shift_range=0.2, # used, Float (fraction of total width). Range for random horizontal shifts.
    height_shift_range=0.2, # used,  Float (fraction of total height). Range for random vertical shifts.
    shear_range=0.1, # Float. Shear Intensity (Shear angle in counter-clockwise direction as radians)
    zoom_range=0.0,
    channel_shift_range=0.,
    fill_mode='nearest',
    cval=0.,
#     brightness_range=[0.2,1],
    horizontal_flip=True,
    vertical_flip=True,
    rescale=None)


datagen_val1 = ImageDataGenerator(
    featurewise_center=False,
    samplewise_center=False,
    featurewise_std_normalization=False,
    samplewise_std_normalization=False,
    zca_whitening=False,
    rotation_range=180, # used, Int. Degree range for random rotations.
    width_shift_range=0.2, # used, Float (fraction of total width). Range for random horizontal shifts.
    height_shift_range=0.2, # used,  Float (fraction of total height). Range for random vertical shifts.
    shear_range=0.1, # Float. Shear Intensity (Shear angle in counter-clockwise direction as radians)
    zoom_range=0.0,
    channel_shift_range=0.,
    fill_mode='nearest',
    cval=0.,
#     brightness_range=[0.2,1],
    horizontal_flip=True,
    vertical_flip=True,
    rescale=None)

datagen_train1.fit(X_train1)
datagen_val1.fit(X_val1)


# %%time
study_name = "resnet_batch"  # Unique identifier of the study.
CV_RESULT_DIR = os.getcwd()+f"/{study_name}/"
if not os.path.exists(CV_RESULT_DIR):  os.mkdir(CV_RESULT_DIR)
storage_name = "sqlite:///{}.db".format(study_name)
import json 

def objective(trial):
    param = {
        "learning_rate": trial.suggest_float("learning_rate",1e-8,1,log=True),
        "beta_1": trial.suggest_float("beta_1",1e-8,1,log=True),
        "beta_2": trial.suggest_float("beta_2",1e-8,1,log=True),
        "epsilon" : trial.suggest_float("epsilon",1e-8,1,log=True),
        "l2_lambda": trial.suggest_float("l2_lambda", 1e-8,1, log=True),
        "dropout": trial.suggest_float("dropout", 0.1,0.99),
        "img_wh": 224,
            }
    resnet_batch2 = build_net1(**param)    
    BS  = 128
    epochs = 25
    history = resnet_batch2.fit_generator(datagen_train1.flow(X_train1, Y_train1, batch_size=BS), 
                   steps_per_epoch=len(X_train1)//BS, # how many generators to go through per epoch
                   epochs=epochs, verbose=1,
                   validation_data=datagen_val1.flow(X_val1,Y_val1, batch_size=BS),
                   validation_steps=len(X_val1)//BS,
                   callbacks=[EarlyStopping(monitor='val_loss', patience=5)],
                  )
#     clf = LineSearchLogisticRegressionMulit(**param)
#     clf.fit(x_trains,y_trains)
#     yhat = clf.predict(x_trainsv)
#     acc = accuracy_score(y_trainsv,yhat)
    print(history.history)
    with open(CV_RESULT_DIR+f"{trial.number}.json", "w") as outfile:
        json.dump(history.history, outfile)
#     with open(CV_RESULT_DIR+f'trial_{trial.number}.txt','w') as data: 
#         data.write(str(history.history))
    
    print(history.history['accuracy'][-1])
    return history.history['accuracy'][-1]


# pruner = optuna.pruners.MedianPruner(n_warmup_steps=5)
# pruner = optuna.pruners.HyperbandPruner()
study = optuna.create_study(direction="maximize",storage=storage_name,study_name=study_name)
study.optimize(objective, n_trials=50)

print("Best trial:")
trial = study.best_trial

print("  Value: {}".format(trial.value))

print("  Params: ")
for key, value in trial.params.items():
    print("    {}: {}".format(key, value))

Plot the Hyperparameter optmization plotes

In [ ]:
from optuna.visualization import plot_contour
from optuna.visualization import plot_edf
from optuna.visualization import plot_intermediate_values
from optuna.visualization import plot_optimization_history
from optuna.visualization import plot_parallel_coordinate
from optuna.visualization import plot_param_importances
from optuna.visualization import plot_slice
loaded_study = optuna.load_study(study_name="resnet_batch", storage="sqlite:///resnet_batch.db")
plot_optimization_history(loaded_study).write_image("resnet_batch.png")
plot_slice(loaded_study).write_image("plot_slice_resnet_batch.png")
plot_parallel_coordinate(loaded_study).write_image("plot_parallel_coordinate_resnet_batch.png")
plot_param_importances(loaded_study).write_image("plot_param_importances_resnet_batch.png")